From f54e72cec06f21f4af939c70541e8a339b7e56ff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 14:45:09 +0200 Subject: first proof of concept on zipped file writer This DOES NOT work sufficiently well, I just wanted to verify that zip writing is possible and files are readable. Will be refined soon. --- configure.ac | 1 + runtime/Makefile.am | 11 ++ runtime/rsyslog.h | 1 + runtime/unicode-helper.h | 1 + runtime/zlibw.c | 119 ++++++++++++++++++++++ runtime/zlibw.h | 45 +++++++++ tools/Makefile.am | 4 +- tools/omfile.c | 234 ++++++++++++++++++++++++++++++++++++------- tools/zpipe.c | 254 +++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 631 insertions(+), 39 deletions(-) create mode 100644 runtime/zlibw.c create mode 100644 runtime/zlibw.h create mode 100644 tools/zpipe.c diff --git a/configure.ac b/configure.ac index 70eea038..4a040e6a 100644 --- a/configure.ac +++ b/configure.ac @@ -186,6 +186,7 @@ AC_ARG_ENABLE(zlib, esac], [enable_zlib=yes] ) +AM_CONDITIONAL(ENABLE_ZLIB, test x$enable_zlib = xyes) if test "$enable_zlib" = "yes"; then AC_CHECK_HEADER(zlib.h, [zlib_header="yes"], [zlib_header="no" enable_zlib="false"]) if test "$zlib_header" = "yes"; then diff --git a/runtime/Makefile.am b/runtime/Makefile.am index eeb656d6..81422d64 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -105,6 +105,17 @@ lmregexp_la_LDFLAGS = -module -avoid-version lmregexp_la_LIBADD = endif +# +# zlib support +# +if ENABLE_ZLIB +pkglib_LTLIBRARIES += lmzlibw.la +lmzlibw_la_SOURCES = zlibw.c zlibw.h +lmzlibw_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmzlibw_la_LDFLAGS = -module -avoid-version +lmzlibw_la_LIBADD = +endif + if ENABLE_INET pkglib_LTLIBRARIES += lmnet.la lmnetstrms.la # diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 293e559b..57c5dcdf 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -275,6 +275,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ RS_RET_NONFATAL_CONFIG_ERR = -2123, /**< non-fatal error during config processing */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ + RS_RET_ZLIB_ERR = -2141, /**< error during zlib call */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 36d76a78..fe4cc1f8 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -48,6 +48,7 @@ static inline uchar* ustrdup(uchar *psz) #define UCHAR_CONSTANT(x) ((uchar*) (x)) +#define CHAR_CONVERT(x) ((char*) (x)) #endif /* multi-include protection */ /* vim:set ai: diff --git a/runtime/zlibw.c b/runtime/zlibw.c new file mode 100644 index 00000000..95ac9138 --- /dev/null +++ b/runtime/zlibw.c @@ -0,0 +1,119 @@ +/* The zlibwrap object. + * + * This is an rsyslog object wrapper around zlib. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "zlibw.h" + +MODULE_TYPE_LIB + +/* static data */ +DEFobjStaticHelpers + + +/* ------------------------------ methods ------------------------------ */ + +/* zlib make strong use of macros for its interface functions, so we can not simply + * pass function pointers to them. Instead, we create very small wrappers which call + * the relevant entry points. + */ + +static int myDeflateInit(z_streamp strm, int level) +{ + return deflateInit(strm, level); +} + +static int myDeflate(z_streamp strm, int flush) +{ + return deflate(strm, flush); +} + +static int myDeflateEnd(z_streamp strm) +{ + return deflateEnd(strm); +} + + +/* queryInterface function + * rgerhards, 2008-03-05 + */ +BEGINobjQueryInterface(zlibw) +CODESTARTobjQueryInterface(zlibw) + if(pIf->ifVersion != zlibwCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DeflateInit = myDeflateInit; + pIf->Deflate = myDeflate; + pIf->DeflateEnd = myDeflateEnd; +finalize_it: +ENDobjQueryInterface(zlibw) + + +/* Initialize the zlibw class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINAbstractObjClassInit(zlibw, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ + /* request objects we use */ + + /* set our own handlers */ +ENDObjClassInit(zlibw) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + CHKiRet(zlibwClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + /* Initialize all classes that are in our module - this includes ourselfs */ +ENDmodInit +/* vi:set ai: + */ diff --git a/runtime/zlibw.h b/runtime/zlibw.h new file mode 100644 index 00000000..01e0b054 --- /dev/null +++ b/runtime/zlibw.h @@ -0,0 +1,45 @@ +/* The zlibw object. It encapsulates the zlib functionality. The primary + * purpose of this wrapper class is to enable rsyslogd core to be build without + * zlib libraries. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_ZLIBW_H +#define INCLUDED_ZLIBW_H + +#include + +/* interfaces */ +BEGINinterface(zlibw) /* name must also be changed in ENDinterface macro! */ + int (*DeflateInit)(z_streamp strm, int); + int (*Deflate)(z_streamp strm, int); + int (*DeflateEnd)(z_streamp strm); +ENDinterface(zlibw) +#define zlibwCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(zlibw); + +/* the name of our library binary */ +#define LM_ZLIBW_FILENAME "lmzlibw" + +#endif /* #ifndef INCLUDED_ZLIBW_H */ diff --git a/tools/Makefile.am b/tools/Makefile.am index e523b854..f0f9afab 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -26,8 +26,10 @@ rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS -sbin_PROGRAMS += rsyslog_diag_hostname msggen +sbin_PROGRAMS += rsyslog_diag_hostname msggen zpipe rsyslog_diag_hostname_SOURCES = gethostn.c +zpipe_SOURCES = zpipe.c +zpipe_LDADD = -lz msggen_SOURCES = msggen.c endif diff --git a/tools/omfile.c b/tools/omfile.c index c7283e4d..62a044d8 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -12,7 +12,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -57,6 +57,8 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "unicode-helper.h" +#include "zlibw.h" MODULE_TYPE_OUTPUT @@ -64,16 +66,30 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +DEFobjCurrIf(zlibw) /* The following structure is a dynafile name cache entry. */ struct s_dynaFileCacheEntry { - uchar *pName; /* name currently open, if dynamic name */ + uchar *pName; /* name currently open, if dynamic name */ short fd; /* name associated with file name in cache */ time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; +/* the output buffer structure */ +// TODO: later make this part of the dynafile cache! +#define OUTBUF_LEN 128 // TODO: make dynamic! +typedef struct { + uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ + size_t lenBuf; /* max size of buffer */ + size_t iBuf; /* current buffer index */ + int fd; /* which file descriptor is this buf for? */ + /* elements for zip writing */ + z_stream zStrm; + char zipBuf[OUTBUF_LEN]; +} outbuf_t; + /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -92,7 +108,8 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ - short fd; /* file descriptor for (current) file */ + short fd; /* file descriptor for (current) file */ + outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -132,7 +149,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - printf("[dynamic]\n\ttemplate='%s'" + dbgprintf("[dynamic]\n\ttemplate='%s'" "\tfile cache size=%d\n" "\tcreate directories: %s\n" "\tfile owner %d, group %d\n" @@ -146,9 +163,9 @@ CODESTARTdbgPrintInstInfo pData->bFailOnChown ? "yes" : "no" ); } else { /* regular file */ - printf("%s", pData->f_fname); + dbgprintf("%s", pData->f_fname); if (pData->fd == -1) - printf(" (unused)"); + dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -321,7 +338,8 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) +static void +dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { ASSERT(pCache != NULL); @@ -356,7 +374,8 @@ finalize_it: * relevant files. Part of Shutdown and HUP processing. * rgerhards, 2008-10-23 */ -static inline void dynaFileFreeCacheEntries(instanceData *pData) +static inline void +dynaFileFreeCacheEntries(instanceData *pData) { register int i; ASSERT(pData != NULL); @@ -562,48 +581,32 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } -/* rgerhards 2004-11-11: write to a file output. This - * will be called for all outputs using file semantics, - * for example also for pipes. +/* physically write the file */ -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal +doPhysWrite(instanceData *pData, int fd, char *pszBuf, size_t lenBuf) { off_t actualFileSize; int iLenWritten; DEFiRet; - ASSERT(pData != NULL); - /* first check if we have a dynamic file name and, if so, - * check if it still is ok or a new file needs to be created - */ - if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - /* create the message based on format specified */ +dbgprintf("doPhysWrite, fd %d, iBuf %d\n", fd, (int) lenBuf); again: /* check if we have a file size limit and, if so, * obey to it. */ if(pData->f_sizeLimit != 0) { - actualFileSize = lseek(pData->fd, 0, SEEK_END); + actualFileSize = lseek(fd, 0, SEEK_END); if(actualFileSize >= pData->f_sizeLimit) { char errMsg[256]; /* for now, we simply disable a file once it is * beyond the maximum size. This is better than having * us aborted by the OS... rgerhards 2005-06-21 */ - (void) close(pData->fd); + (void) close(fd); /* try to resolve the situation */ + // TODO: *doesn't work, will need to use new fd ! if(resolveFileSizeLimit(pData) != 0) { /* didn't work out, so disable... */ snprintf(errMsg, sizeof(errMsg), @@ -622,13 +625,12 @@ again: } } - iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); -//dbgprintf("lenwritten: %d\n", iLenWritten); + iLenWritten = write(fd, pszBuf, lenBuf); if(iLenWritten < 0) { int e = errno; char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); + DBGPRINTF("log file (%d) write error %d: %s\n", fd, e, errStr); /* If a named pipe is full, we suspend this action for a while */ if(pData->fileType == eTypePIPE && e == EAGAIN) @@ -667,9 +669,159 @@ again: errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } } else if (pData->bSyncFile) { - fsync(pData->fd); + fsync(fd); + } + + pData->poBuf->iBuf = 0; + +finalize_it: + RETiRet; +} + + +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + */ +static rsRetVal +doZipWrite(instanceData *pData) +{ + outbuf_t *poBuf; + z_stream strm; + int zRet; /* zlib return state */ + DEFiRet; + assert(pData != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ + strm = poBuf->zStrm; /* another shortcut */ + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + zRet = zlibw.DeflateInit(&strm, 9); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateInit()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateInit() done successfully\n"); + + /* now doing the compression */ + strm.avail_in = poBuf->iBuf; + strm.next_in = (Bytef*) poBuf->pszBuf; + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", strm.avail_in, strm.total_in); + strm.avail_out = OUTBUF_LEN; + strm.next_out = (Bytef*) poBuf->zipBuf; + zRet = zlibw.Deflate(&strm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, strm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - strm.avail_out)); + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + +RUNLOG_STR("deflate() should be done successfully\n"); + + zRet = zlibw.DeflateEnd(&strm); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateEnd() done successfully\n"); + +finalize_it: + RETiRet; +} + + +/* flush the output buffer + */ +static rsRetVal +doFlush(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + + if(pData->poBuf->iBuf == 0) + FINALIZE; /* nothing to write, but make this a valid case */ + + if(1) { // zlib enabled! + CHKiRet(doZipWrite(pData)); + } else { + CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); + } + +finalize_it: + RETiRet; +} + + +/* do the actual write process. This function is to be called once we are ready for writing. + * It will do buffered writes and persist data only when the buffer is full. Note that we must + * be careful to detect when the file handle changed. + * rgerhards, 2009-06-03 + */ +static rsRetVal +doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) +{ + int i; + outbuf_t *poBuf; + DEFiRet; + ASSERT(pData != NULL); + ASSERT(pszBuf != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ +dbgprintf("doWrite, pData->fd %d, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", +pData->fd, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); + + if(pData->fd != poBuf->fd) { + // TODO: more efficient use for dynafiles + CHKiRet(doFlush(pData)); + poBuf->fd = pData->fd; + } + + for(i = 0 ; i < lenBuf ; ++i) { + poBuf->pszBuf[poBuf->iBuf++] = pszBuf[i]; + if(poBuf->iBuf == poBuf->lenBuf) { + CHKiRet(doFlush(pData)); + } + } + +finalize_it: + RETiRet; +} + + +/* rgerhards 2004-11-11: write to a file output. This + * will be called for all outputs using file semantics, + * for example also for pipes. + */ +static rsRetVal +writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +{ + DEFiRet; + + ASSERT(pData != NULL); + + /* first check if we have a dynamic file name and, if so, + * check if it still is ok or a new file needs to be created + */ + if(pData->bDynamicName) { + if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + } + + if(pData->fd == -1) { + rsRetVal iRetLocal; + iRetLocal = prepareFile(pData, pData->f_fname); + if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } + /* create the message based on format specified */ + CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); + finalize_it: RETiRet; } @@ -678,15 +830,20 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->fd = -1; + CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); + pData->poBuf->lenBuf = OUTBUF_LEN; +finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance + doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->fd != -1) close(pData->fd); + free(pData->poBuf); ENDfreeInstance @@ -696,7 +853,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->f_fname); + DBGPRINTF("file to log to: %s\n", pData->f_fname); iRet = writeFile(ppString, iMsgOpts, pData); ENDdoAction @@ -875,8 +1032,8 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit - if(pszTplName != NULL) - free(pszTplName); + objRelease(zlibw, LM_ZLIBW_FILENAME); + free(pszTplName); ENDmodExit @@ -891,6 +1048,7 @@ BEGINmodInit(File) CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); diff --git a/tools/zpipe.c b/tools/zpipe.c new file mode 100644 index 00000000..bde6c5c1 --- /dev/null +++ b/tools/zpipe.c @@ -0,0 +1,254 @@ +/* zpipe.c: example of proper use of zlib's inflate() and deflate() + Not copyrighted -- provided to the public domain + Version 1.5 11 December 2005 Mark Adler + Version 2.0 03 June 2009 Rainer Gerhards */ + +/* RSYSLOG NOTE: + * This file is beeing distributed as part of rsyslog, but is just an + * add-on. Most importantly, rsyslog's copyright does not apply but + * rather the (non-) copyright stated above. + */ + +/* Version history: + 1.0 30 Oct 2004 First version + 1.1 8 Nov 2004 Add void casting for unused return values + Use switch statement for inflate() return values + 1.2 9 Nov 2004 Add assertions to document zlib guarantees + 1.3 6 Apr 2005 Remove incorrect assertion in inf() + 1.4 11 Dec 2005 Add hack to avoid MSDOS end-of-line conversions + Avoid some compiler warnings for input and output buffers + 2.0 03 Jun 2009 Add hack to support multiple deflate records inside a single + file on inflate. This is needed in order to support reading + files created by rsyslog's zip output writer. + */ + +#include +#include +#include +#include "zlib.h" + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include +# include +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 + +/* Compress from file source to file dest until EOF on source. + def() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_STREAM_ERROR if an invalid compression + level is supplied, Z_VERSION_ERROR if the version of zlib.h and the + version of the library linked do not match, or Z_ERRNO if there is + an error reading or writing the files. */ +int def(FILE *source, FILE *dest, int level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, level); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + flush = feof(source) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + return Z_OK; +} + + +/* initialize stream for deflating (we need this in case of + * multiple records. + * rgerhards, 2009-06-03 + */ +int doInflateInit(z_stream *strm) +{ + int ret; + + /* allocate inflate state */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + strm->avail_in = 0; + strm->next_in = Z_NULL; + ret = inflateInit(strm); + return ret; +} + + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ +int inf(FILE *source, FILE *dest) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + int len; + unsigned char *next_in_save; + unsigned char out[CHUNK]; + + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + len = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + if (len == 0) { + break; + } + strm.avail_in = len; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + strm.avail_out = CHUNK; + strm.next_out = out; + do { + /* fprintf(stderr, "---inner LOOP---, avail_in %d, avail_out %d Byte 0: %x, 1: %x\n", strm.avail_in, strm.avail_out, *strm.next_in, *(strm.next_in+1));*/ + do { + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + /* handle the case that more than one deflate record is contained + * in a single file. -- rgerhards, 2009-06-03 + */ + if(ret == Z_STREAM_END) { + len -= strm.total_in; + if(len > 0) { + next_in_save = strm.next_in; + (void)inflateEnd(&strm); + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + strm.avail_in = len; + strm.next_in = next_in_save; + strm.avail_out = CHUNK; + strm.next_out = out; + ret = Z_OK; /* continue outer loop */ + } + } + } while (strm.avail_in > 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + +/* report a zlib or i/o error */ +void zerr(int ret) +{ + fputs("zpipe: ", stdout); + switch (ret) { + case Z_ERRNO: + if (ferror(stdin)) + fputs("error reading stdin\n", stdout); + if (ferror(stdout)) + fputs("error writing stdout\n", stdout); + break; + case Z_STREAM_ERROR: + fputs("invalid compression level\n", stdout); + break; + case Z_DATA_ERROR: + fputs("invalid or incomplete deflate data\n", stdout); + break; + case Z_MEM_ERROR: + fputs("out of memory\n", stdout); + break; + case Z_VERSION_ERROR: + fputs("zlib version mismatch!\n", stdout); + } +} + +/* compress or decompress from stdin to stdout */ +int main(int argc, char **argv) +{ + int ret; + + /* avoid end-of-line conversions */ + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + + /* do compression if no arguments */ + if (argc == 1) { + ret = def(stdin, stdout, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* do decompression if -d specified */ + else if (argc == 2 && strcmp(argv[1], "-d") == 0) { + ret = inf(stdin, stdout); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* otherwise, report usage */ + else { + fputs("zpipe usage: zpipe [-d] < source > dest\n", stdout); + return 1; + } +} -- cgit From 69eac5341d49c5c7278e6d86a77dc9d6b4a4edde Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 15:32:39 +0200 Subject: added gzip header to output files so they can now be processed with the "regular" gzip tools --- runtime/zlibw.c | 10 ++++++++-- runtime/zlibw.h | 1 + tools/omfile.c | 30 +++++++++++++++++++++++++++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/runtime/zlibw.c b/runtime/zlibw.c index 95ac9138..2b386213 100644 --- a/runtime/zlibw.c +++ b/runtime/zlibw.c @@ -51,9 +51,9 @@ static int myDeflateInit(z_streamp strm, int level) return deflateInit(strm, level); } -static int myDeflate(z_streamp strm, int flush) +static int myDeflateInit2(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy) { - return deflate(strm, flush); + return deflateInit2(strm, level, method, windowBits, memLevel, strategy); } static int myDeflateEnd(z_streamp strm) @@ -61,6 +61,11 @@ static int myDeflateEnd(z_streamp strm) return deflateEnd(strm); } +static int myDeflate(z_streamp strm, int flush) +{ + return deflate(strm, flush); +} + /* queryInterface function * rgerhards, 2008-03-05 @@ -77,6 +82,7 @@ CODESTARTobjQueryInterface(zlibw) * of course, also affects the "if" above). */ pIf->DeflateInit = myDeflateInit; + pIf->DeflateInit2 = myDeflateInit2; pIf->Deflate = myDeflate; pIf->DeflateEnd = myDeflateEnd; finalize_it: diff --git a/runtime/zlibw.h b/runtime/zlibw.h index 01e0b054..63d8f386 100644 --- a/runtime/zlibw.h +++ b/runtime/zlibw.h @@ -30,6 +30,7 @@ /* interfaces */ BEGINinterface(zlibw) /* name must also be changed in ENDinterface macro! */ int (*DeflateInit)(z_streamp strm, int); + int (*DeflateInit2)(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy); int (*Deflate)(z_streamp strm, int); int (*DeflateEnd)(z_streamp strm); ENDinterface(zlibw) diff --git a/tools/omfile.c b/tools/omfile.c index 62a044d8..08c33a7f 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -14,6 +14,29 @@ * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * + * An important note on writing gzip format via zlib (kept anonymous + * by request): + * + * -------------------------------------------------------------------------- + * We'd like to make sure the output file is in full gzip format + * (compatible with gzip -d/zcat etc). There is a flag in how the output + * is initialized within zlib to properly add the gzip wrappers to the + * output. (gzip is effectively a small metadata wrapper around raw + * zstream output.) + * + * I had written an old bit of code to do this - the documentation on + * deflatInit2() was pretty tricky to nail down on this specific feature: + * + * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, + * int memLevel, int strategy); + * + * I believe "31" would be the value for the "windowBits" field that you'd + * want to try: + * + * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + * -------------------------------------------------------------------------- + * + * * This file is part of rsyslog. * * Rsyslog is free software: you can redistribute it and/or modify @@ -698,12 +721,13 @@ doZipWrite(instanceData *pData) strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; - zRet = zlibw.DeflateInit(&strm, 9); + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&strm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateInit()\n", zRet); + dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } -RUNLOG_STR("deflateInit() done successfully\n"); +RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ strm.avail_in = poBuf->iBuf; -- cgit From e893a5916473c45d6b90dc2401637fe713133291 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 10:32:12 +0200 Subject: backported macros necessary to create output modules with transaction support this does not mean the v4 engine will support batches and output transactions, but I can now write plugins that will work equally well with v4 AND v5. I consider this useful during the rewrite of omfile. --- runtime/module-template.h | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/runtime/module-template.h b/runtime/module-template.h index 6f7d877c..3e963199 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -39,7 +39,8 @@ #define DEF_OMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ - DEFobjCurrIf(obj) + DEFobjCurrIf(obj) \ + static __attribute__((unused)) int bCoreSupportsBatching; #define DEF_IMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ DEFobjCurrIf(obj) @@ -160,6 +161,37 @@ static rsRetVal isCompatibleWithFeature(syslogFeature __attribute__((unused)) eF RETiRet;\ } + +/* beginTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINbeginTransaction \ +static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTbeginTransaction /* currently empty, but may be extended */ + +#define ENDbeginTransaction \ + RETiRet;\ +} + + +/* endTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINendTransaction \ +static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTendTransaction /* currently empty, but may be extended */ + +#define ENDendTransaction \ + RETiRet;\ +} + + /* doAction() */ #define BEGINdoAction \ @@ -324,6 +356,18 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = tryResume;\ } + +/* the following definition is queryEtryPt block that must be added + * if an output module supports the transactional interface. + * rgerhards, 2009-04-27 + */ +#define CODEqueryEtryPt_TXIF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "beginTransaction")) {\ + *pEtryPoint = beginTransaction;\ + } else if(!strcmp((char*) name, "endTransaction")) {\ + *pEtryPoint = endTransaction;\ + } + /* the following definition is the standard block for queryEtryPt for INPUT * modules. This can be used if no specific handling (e.g. to cover version * differences) is needed. @@ -393,6 +437,32 @@ finalize_it:\ } +/* now come some check functions, which enable a standard way of obtaining feature + * information from the core. feat is the to-be-tested feature and featVar is a + * variable that receives the result (0-not support, 1-supported). + * This must be a macro, so that it is put into the output's code. Otherwise, we + * would need to rely on a library entry point, which is what we intend to avoid ;) + * rgerhards, 2009-04-27 + */ +#define INITChkCoreFeature(featVar, feat) \ +{ \ + rsRetVal MACRO_Ret; \ + rsRetVal (*pQueryCoreFeatureSupport)(int*, unsigned); \ + int bSupportsIt; \ + featVar = 0; \ + MACRO_Ret = pHostQueryEtryPt((uchar*)"queryCoreFeatureSupport", &pQueryCoreFeatureSupport); \ + if(MACRO_Ret == RS_RET_OK) { \ + /* found entry point, so let's see if core supports it */ \ + CHKiRet((*pQueryCoreFeatureSupport)(&bSupportsIt, feat)); \ + if(bSupportsIt) \ + featVar = 1; \ + } else if(MACRO_Ret != RS_RET_ENTRY_POINT_NOT_FOUND) { \ + ABORT_FINALIZE(MACRO_Ret); /* Something else went wrong, what is not acceptable */ \ + } \ +} + + + /* definitions for host API queries */ #define CODEmodInit_QueryRegCFSLineHdlr \ CHKiRet(pHostQueryEtryPt((uchar*)"regCfSysLineHdlr", &omsdRegCFSLineHdlr)); -- cgit From 768836ab79268e1091fc3af8bb920c03287db91a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 11:05:41 +0200 Subject: reduced max value for $DynaFileCacheSize to 1,000 The former maximum of 10,000 really made no sense, even 1,000 is very high, but we like to keep the user in control ;)). Also, some general cleanup was done. --- ChangeLog | 3 +++ tools/omfile.c | 66 +++++++++++++++++++++++++--------------------------------- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/ChangeLog b/ChangeLog index f9e3e32b..9d398f09 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,9 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - bugfix: imdiag/imtcp had a race condition - improved testbench (now much better code design and reuse) - added config switch --enable-testbench=no to turn off testbench +- reduced max value for $DynaFileCacheSize to 1,000 (the former maximum + of 10,000 really made no sense, even 1,000 is very high, but we like + to keep the user in control ;)). --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/tools/omfile.c b/tools/omfile.c index 08c33a7f..d1274f1b 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -12,6 +12,12 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * + * A large re-write of this file was done in June, 2009. The focus was + * to introduce many more features (like zipped writing), clean up the code + * and make it more reliable. In short, that rewrite tries to provide a new + * solid basis for the next three to five years to come. During it, bugs + * may have been introduced ;) -- rgerhards, 2009-06-04 + * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * An important note on writing gzip format via zlib (kept anonymous @@ -158,7 +164,7 @@ typedef struct _instanceData { */ dynaFileCacheEntry **dynCache; off_t f_sizeLimit; /* file size limit, 0 = no limit */ - char *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ } instanceData; @@ -208,13 +214,13 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal) errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; iNewVal = 1; - } else if(iNewVal > 10000) { + } else if(iNewVal > 1000) { snprintf((char*) errMsg, sizeof(errMsg)/sizeof(uchar), - "DynaFileCacheSize maximum is 10,000 (%d given), changed to 10,000.", iNewVal); + "DynaFileCacheSize maximum is 1,000 (%d given), changed to 1,000.", iNewVal); errno = 0; errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; - iNewVal = 10000; + iNewVal = 1000; } iDynaFileCacheSize = iNewVal; @@ -286,7 +292,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit; + pData->f_sizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -322,7 +328,7 @@ int resolveFileSizeLimit(instanceData *pData) * when we have a space in the program name. If we find it, everything after * the space is treated as a single argument. */ - if((pCmd = (uchar*)strdup((char*)pData->f_sizeLimitCmd)) == NULL) { + if((pCmd = ustrdup(pData->f_sizeLimitCmd)) == NULL) { /* there is not much we can do - we make syslogd close the file in this case */ return 1; } @@ -372,7 +378,7 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) FINALIZE; DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, - pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName); + pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName); /* if the name is NULL, this is an improperly initilized entry which * needs to be discarded. In this case, neither the file is to be closed * not the name to be freed. @@ -452,7 +458,7 @@ prepareFile(instanceData *pData, uchar *newFileName) * We do not report any errors here ourselfs but let the code * fall through to error handler below. */ - if(makeFileParentDirs(newFileName, strlen((char*)newFileName), + if(makeFileParentDirs(newFileName, ustrlen(newFileName), pData->fDirCreateMode, pData->dirUID, pData->dirGID, pData->bFailOnChown) != 0) { ABORT_FINALIZE(RS_RET_ERR); /* we give up */ @@ -523,7 +529,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg * I *hope* this will be a performance enhancement. */ if( (pData->iCurrElt != -1) - && !strcmp((char*) newFileName, (char*) pCache[pData->iCurrElt]->pName)) { + && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ return 0; @@ -540,7 +546,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if(iFirstFree == -1) iFirstFree = i; } else { /* got an element, let's see if it matches */ - if(!strcmp((char*) newFileName, (char*) pCache[i]->pName)) { + if(!ustrcmp(newFileName, pCache[i]->pName)) { /* we found our element! */ pData->fd = pCache[i]->fd; pData->iCurrElt = i; @@ -585,7 +591,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if(iMsgOpts & INTERNAL_MSG) { DBGPRINTF("Could not open dynaFile, discarding message\n"); } else { - errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName); + errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } dynaFileDelCacheEntry(pCache, iFirstFree, 1); pData->iCurrElt = -1; @@ -593,7 +599,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } pCache[iFirstFree]->fd = pData->fd; - pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */ + pCache[iFirstFree]->pName = ustrdup(newFileName); /* TODO: check for NULL (very unlikely) */ pCache[iFirstFree]->lastUsed = time(NULL); pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); @@ -884,21 +890,10 @@ ENDdoAction BEGINparseSelectorAct CODESTARTparseSelectorAct - /* yes, the if below is redundant, but I need it now. Will go away as - * the code further changes. -- rgerhards, 2007-07-25 - */ - if(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-') { - if((iRet = createInstance(&pData)) != RS_RET_OK) { - ENDfunc - return iRet; /* this can not use RET_iRet! */ - } - } else { - /* this is not clean, but we need it for the time being - * TODO: remove when cleaning up modularization - */ - ENDfunc - return RS_RET_CONFLINE_UNPROCESSED; - } + if(!(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-')) + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + + CHKiRet(createInstance(&pData)); if(*p == '-') { pData->bSyncFile = 0; @@ -933,15 +928,12 @@ CODESTARTparseSelectorAct */ CODE_STD_STRING_REQUESTparseSelectorAct(2) ++p; /* eat '?' */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); /* "filename" is actually a template name, we need this as string 1. So let's add it * to the pOMSR. -- rgerhards, 2007-07-27 */ - if((iRet = OMSRsetEntry(*ppOMSR, 1, (uchar*)strdup((char*) pData->f_fname), OMSR_NO_RQD_TPL_OPTS)) != RS_RET_OK) - break; + CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(pData->f_fname), OMSR_NO_RQD_TPL_OPTS)); pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ @@ -979,10 +971,8 @@ CODESTARTparseSelectorAct * to use is specified. So we need to scan for the first coma first * and then look at the rest of the line. */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); pData->bDynamicName = 0; pData->fCreateMode = fCreateMode; /* preserve current setting */ @@ -1005,7 +995,7 @@ CODESTARTparseSelectorAct errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); break; } - if(strcmp((char*) p, _PATH_CONSOLE) == 0) + if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) pData->fileType = eTypeCONSOLE; break; default: -- cgit From 9e434f19a9baa4a6f411808b5cb6bc22d6a32781 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 12:15:59 +0200 Subject: cleaned up stream class ... ... and also made it callable via an rsyslog interface rather then relying on the OS loader (important if we go for using it inside loadbale modules, which we soon possible will) --- plugins/imfile/imfile.c | 51 ++++++++++++------------ runtime/obj.c | 64 ++++++++++++++++--------------- runtime/queue.c | 90 ++++++++++++++++++++++--------------------- runtime/rsyslog.c | 2 - runtime/stream.c | 100 ++++++++++++++++++++++++++++++------------------ runtime/stream.h | 56 +++++++++++++-------------- runtime/sysvar.c | 2 - tools/omfile.c | 2 +- tools/syslogd.c | 21 ---------- 9 files changed, 198 insertions(+), 190 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 92fd30c3..ff32c857 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -56,6 +56,7 @@ DEF_IMOD_STATIC_DATA /* must be present, starts static data */ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) +DEFobjCurrIf(strm) typedef struct fileInfo_s { uchar *pszFileName; @@ -139,16 +140,16 @@ openFile(fileInfo_t *pThis) /* If we reach this point, we have a .si file */ - CHKiRet(strmConstruct(&psSF)); - CHKiRet(strmSettOperationsMode(psSF, STREAMMODE_READ)); - CHKiRet(strmSetsType(psSF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psSF, pszSFNam, lenSFNam)); - CHKiRet(strmConstructFinalize(psSF)); + CHKiRet(strm.Construct(&psSF)); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_READ)); + CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psSF, pszSFNam, lenSFNam)); + CHKiRet(strm.ConstructFinalize(psSF)); /* read back in the object */ CHKiRet(obj.Deserialize(&pThis->pStrm, (uchar*) "strm", psSF, NULL, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->pStrm)); + CHKiRet(strm.SeekCurrOffs(pThis->pStrm)); /* OK, we could successfully read the file, so we now can request that it be deleted. * If we need it again, it will be written on the next shutdown. @@ -157,14 +158,14 @@ openFile(fileInfo_t *pThis) finalize_it: if(psSF != NULL) - strmDestruct(&psSF); + strm.Destruct(&psSF); if(iRet != RS_RET_OK) { - CHKiRet(strmConstruct(&pThis->pStrm)); - CHKiRet(strmSettOperationsMode(pThis->pStrm, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR)); - CHKiRet(strmSetFName(pThis->pStrm, pThis->pszFileName, strlen((char*) pThis->pszFileName))); - CHKiRet(strmConstructFinalize(pThis->pStrm)); + CHKiRet(strm.Construct(&pThis->pStrm)); + CHKiRet(strm.SettOperationsMode(pThis->pStrm, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR)); + CHKiRet(strm.SetFName(pThis->pStrm, pThis->pszFileName, strlen((char*) pThis->pszFileName))); + CHKiRet(strm.ConstructFinalize(pThis->pStrm)); } RETiRet; @@ -203,7 +204,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) /* loop below will be exited when strmReadLine() returns EOF */ while(1) { - CHKiRet(strmReadLine(pThis->pStrm, &pCStr)); + CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr)); *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ @@ -354,21 +355,21 @@ persistStrmState(fileInfo_t *pInfo) ASSERT(pInfo != NULL); /* TODO: create a function persistObj in obj.c? */ - CHKiRet(strmConstruct(&psSF)); - CHKiRet(strmSetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSettOperationsMode(psSF, STREAMMODE_WRITE)); - CHKiRet(strmSetiAddtlOpenFlags(psSF, O_TRUNC)); - CHKiRet(strmSetsType(psSF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); - CHKiRet(strmConstructFinalize(psSF)); + CHKiRet(strm.Construct(&psSF)); + CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE)); + CHKiRet(strm.SetiAddtlOpenFlags(psSF, O_TRUNC)); + CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); + CHKiRet(strm.ConstructFinalize(psSF)); - CHKiRet(strmSerialize(pInfo->pStrm, psSF)); + CHKiRet(strm.Serialize(pInfo->pStrm, psSF)); - CHKiRet(strmDestruct(&psSF)); + CHKiRet(strm.Destruct(&psSF)); finalize_it: if(psSF != NULL) - strmDestruct(&psSF); + strm.Destruct(&psSF); RETiRet; } @@ -388,7 +389,7 @@ CODESTARTafterRun for(i = 0 ; i < iFilPtr ; ++i) { if(files[i].pStrm != NULL) { /* stream open? */ persistStrmState(&files[i]); - strmDestruct(&(files[i].pStrm)); + strm.Destruct(&(files[i].pStrm)); } } ENDafterRun @@ -401,6 +402,7 @@ ENDafterRun BEGINmodExit CODESTARTmodExit /* release objects we used */ + objRelease(strm, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); @@ -512,6 +514,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilename", 0, eCmdHdlrGetWord, NULL, &pszFileName, STD_LOADABLE_MODULE_ID)); diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..f5ec66b2 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -93,6 +93,7 @@ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some ma DEFobjCurrIf(var) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +DEFobjCurrIf(strm) static objInfo_t *arrObjInfo[OBJ_NUM_IDS]; /* array with object information pointers */ @@ -228,20 +229,20 @@ static rsRetVal objSerializeHeader(strm_t *pStrm, obj_t *pObj, uchar *pszRecType assert(!strcmp((char*) pszRecType, "Obj") || !strcmp((char*) pszRecType, "OPB")); /* object cookie and serializer version (so far always 1) */ - CHKiRet(strmWriteChar(pStrm, COOKIE_OBJLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '1')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_OBJLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '1')); /* object type, version and string length */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWrite(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteLong(pStrm, objGetVersion(pObj))); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, objGetVersion(pObj))); /* record trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -259,7 +260,7 @@ BeginSerialize(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "Obj")); finalize_it: @@ -284,7 +285,7 @@ BeginSerializePropBag(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "OPB")); finalize_it: @@ -377,23 +378,23 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr } /* cookie */ - CHKiRet(strmWriteChar(pStrm, COOKIE_PROPLINE)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_PROPLINE)); /* name */ - CHKiRet(strmWrite(pStrm, pszPropName, strlen((char*)pszPropName))); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pszPropName, strlen((char*)pszPropName))); + CHKiRet(strm.WriteChar(pStrm, ':')); /* type */ - CHKiRet(strmWriteLong(pStrm, (int) vType)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, (int) vType)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* length */ - CHKiRet(strmWriteLong(pStrm, lenBuf)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, lenBuf)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* data */ - CHKiRet(strmWrite(pStrm, (uchar*) pszBuf, lenBuf)); + CHKiRet(strm.Write(pStrm, (uchar*) pszBuf, lenBuf)); /* trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -410,12 +411,12 @@ EndSerialize(strm_t *pStrm) assert(pStrm != NULL); - CHKiRet(strmWriteChar(pStrm, COOKIE_ENDLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); - CHKiRet(strmWriteChar(pStrm, COOKIE_BLANKLINE)); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_ENDLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_BLANKLINE)); + CHKiRet(strm.WriteChar(pStrm, '\n')); - CHKiRet(strmRecordEnd(pStrm)); + CHKiRet(strm.RecordEnd(pStrm)); finalize_it: RETiRet; @@ -423,7 +424,7 @@ finalize_it: /* define a helper to make code below a bit cleaner (and quicker to write) */ -#define NEXTC CHKiRet(strmReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ +#define NEXTC CHKiRet(strm.ReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ /* de-serialize an embedded, non-octect-counted string. This is useful @@ -617,7 +618,7 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) NEXTC; if(c != COOKIE_PROPLINE) { /* oops, we've read one char that does not belong to use - unget it first */ - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); ABORT_FINALIZE(RS_RET_NO_PROPLINE); } @@ -718,7 +719,7 @@ static rsRetVal objDeserializeTryRecover(strm_t *pStrm) } } - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); finalize_it: dbgprintf("deserializer has possibly been able to re-sync and recover, state %d\n", iRet); @@ -1278,6 +1279,7 @@ objClassExit(void) { DEFiRet; /* release objects we no longer need */ + objRelease(strm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); @@ -1322,9 +1324,11 @@ objClassInit(modInfo_t *pModInfo) CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); CHKiRet(moduleClassInit(pModInfo)); + CHKiRet(strmClassInit(pModInfo)); CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); finalize_it: RETiRet; diff --git a/runtime/queue.c b/runtime/queue.c index 4e017e84..3f14b535 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -59,6 +59,7 @@ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) +DEFobjCurrIf(strm) /* forward-definitions */ rsRetVal qqueueChkPersist(qqueue_t *pThis); @@ -667,7 +668,7 @@ qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pTh DEFiRet; ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_TYPE_assert(pThis, qqueue); - CHKiRet(strmSetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); finalize_it: RETiRet; } @@ -744,11 +745,11 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* If we reach this point, we have a .qi file */ - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_READ)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_READ)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, we try to read the property bag for ourselfs */ CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF)); @@ -770,8 +771,8 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pRead)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pRead)); /* OK, we could successfully read the file, so we now can request that it be * deleted when we are done with the persisted information. @@ -780,7 +781,7 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); if(iRet != RS_RET_OK) { dbgoprint((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", @@ -815,24 +816,24 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) if(bRestarted == 1) { ; } else { - CHKiRet(strmConstruct(&pThis->tVars.disk.pWrite)); - CHKiRet(strmSetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); - CHKiRet(strmSetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pWrite)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); - CHKiRet(strmConstruct(&pThis->tVars.disk.pRead)); - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); - CHKiRet(strmSetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pRead)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pRead)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pRead)); - CHKiRet(strmSetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(strmSetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); } /* now we set (and overwrite in case of a persisted restart) some parameters which @@ -840,8 +841,8 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) * for example file name generation must not be changed as that would break the * ability to read existing queue files. -- rgerhards, 2008-01-12 */ - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); finalize_it: RETiRet; @@ -854,8 +855,8 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) ASSERT(pThis != NULL); - strmDestruct(&pThis->tVars.disk.pWrite); - strmDestruct(&pThis->tVars.disk.pRead); + strm.Destruct(&pThis->tVars.disk.pWrite); + strm.Destruct(&pThis->tVars.disk.pRead); RETiRet; } @@ -867,10 +868,10 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) ASSERT(pThis != NULL); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); CHKiRet((objSerialize(pUsr))(pUsr, pThis->tVars.disk.pWrite)); - CHKiRet(strmFlush(pThis->tVars.disk.pWrite)); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ + CHKiRet(strm.Flush(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ pThis->tVars.disk.sizeOnDisk += nWriteCount; @@ -894,9 +895,9 @@ static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) int64 offsIn; int64 offsOut; - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pRead, NULL, NULL)); - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); /* This time it is a bit tricky: we free disk space only upon file deletion. So we need * to keep track of what we have read until we get an out-offset that is lower than the @@ -1917,16 +1918,16 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); FINALIZE; /* nothing left to do, so be happy */ } - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_WRITE)); - CHKiRet(strmSetiAddtlOpenFlags(psQIF, O_TRUNC)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE)); + CHKiRet(strm.SetiAddtlOpenFlags(psQIF, O_TRUNC)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, write the property bag for ourselfs * And, surprisingly enough, we currently need to persist only the size of the @@ -1951,14 +1952,14 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) } /* now persist the stream info */ - CHKiRet(strmSerialize(pThis->tVars.disk.pWrite, psQIF)); - CHKiRet(strmSerialize(pThis->tVars.disk.pRead, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pRead, psQIF)); /* tell the input file object that it must not delete the file on close if the queue * is non-empty - but only if we are not during a simple checkpoint */ if(bIsCheckpoint != QUEUE_CHECKPOINT) { - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); } /* we have persisted the queue object. So whenever it comes to an empty queue, @@ -1968,7 +1969,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); RETiRet; } @@ -2340,6 +2341,7 @@ rsRetVal qqueueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 8df100a1..81550f12 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -150,8 +150,6 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "str,"; - CHKiRet(strmClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "wti"; CHKiRet(wtiClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "wtp"; diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..261eb9ca 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -45,10 +45,17 @@ #include "srUtils.h" #include "obj.h" #include "stream.h" +#include "unicode-helper.h" +#include "module-template.h" /* static data */ DEFobjStaticHelpers +/* forward definitions */ +static rsRetVal strmFlush(strm_t *pThis); +static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); + + /* methods */ /* first, we define type-specific handlers. The provide a generic functionality, @@ -234,10 +241,6 @@ strmHandleEOF(strm_t *pThis) case STREAMTYPE_FILE_CIRCULAR: /* we have multiple files and need to switch to the next one */ /* TODO: think about emulating EOF in this case (not yet needed) */ -#if 0 - if(pThis->iMaxFiles == 0) /* TODO: why do we need this? ;) */ - ABORT_FINALIZE(RS_RET_EOF); -#endif dbgoprint((obj_t*) pThis, "file %d EOF\n", pThis->fd); CHKiRet(strmNextFile(pThis)); break; @@ -295,7 +298,7 @@ finalize_it: * NOTE: needs to be enhanced to support sticking with a strm entry (if not * deleted). */ -rsRetVal strmReadChar(strm_t *pThis, uchar *pC) +static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) { DEFiRet; @@ -329,7 +332,7 @@ finalize_it: * character buffering capability. * rgerhards, 2008-01-07 */ -rsRetVal strmUnreadChar(strm_t *pThis, uchar c) +static rsRetVal strmUnreadChar(strm_t *pThis, uchar c) { ASSERT(pThis != NULL); ASSERT(pThis->iUngetC == -1); @@ -351,7 +354,7 @@ rsRetVal strmUnreadChar(strm_t *pThis, uchar c) * are pthread_killed() upon termination. So if we use their native pointer, they * can cleanup (but only then). */ -rsRetVal +static rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr) { DEFiRet; @@ -393,7 +396,7 @@ ENDobjConstruct(strm) /* ConstructionFinalizer * rgerhards, 2008-01-09 */ -rsRetVal strmConstructFinalize(strm_t *pThis) +static rsRetVal strmConstructFinalize(strm_t *pThis) { DEFiRet; @@ -420,14 +423,10 @@ CODESTARTobjDestruct(strm) if(pThis->fd != -1) strmCloseFile(pThis); - if(pThis->pszDir != NULL) - free(pThis->pszDir); - if(pThis->pIOBuf != NULL) - free(pThis->pIOBuf); - if(pThis->pszCurrFName != NULL) - free(pThis->pszCurrFName); - if(pThis->pszFName != NULL) - free(pThis->pszFName); + free(pThis->pszDir); + free(pThis->pIOBuf); + free(pThis->pszCurrFName); + free(pThis->pszFName); ENDobjDestruct(strm) @@ -453,6 +452,7 @@ finalize_it: RETiRet; } + /* write memory buffer to a stream object. * To support direct writes of large objects, this method may be called * with a buffer pointing to some region other than the stream buffer itself. @@ -503,7 +503,8 @@ finalize_it: * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 */ -rsRetVal strmFlush(strm_t *pThis) +static rsRetVal +strmFlush(strm_t *pThis) { DEFiRet; @@ -545,7 +546,7 @@ static rsRetVal strmSeek(strm_t *pThis, off_t offs) /* seek to current offset. This is primarily a helper to readjust the OS file * pointer after a strm object has been deserialized. */ -rsRetVal strmSeekCurrOffs(strm_t *pThis) +static rsRetVal strmSeekCurrOffs(strm_t *pThis) { DEFiRet; @@ -558,7 +559,7 @@ rsRetVal strmSeekCurrOffs(strm_t *pThis) /* write a *single* character to a stream object -- rgerhards, 2008-01-10 */ -rsRetVal strmWriteChar(strm_t *pThis, uchar c) +static rsRetVal strmWriteChar(strm_t *pThis, uchar c) { DEFiRet; @@ -578,7 +579,7 @@ finalize_it: /* write an integer value (actually a long) to a stream object */ -rsRetVal strmWriteLong(strm_t *pThis, long i) +static rsRetVal strmWriteLong(strm_t *pThis, long i) { DEFiRet; uchar szBuf[32]; @@ -595,7 +596,8 @@ finalize_it: /* write memory buffer to a stream object */ -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static rsRetVal +strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; size_t iPartial; @@ -645,14 +647,14 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) -rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) +static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { pThis->iMaxFiles = iNewVal; pThis->iFileNumDigits = getNumberDigits(iNewVal); return RS_RET_OK; } -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) +static rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) { DEFiRet; @@ -671,7 +673,7 @@ finalize_it: * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) { DEFiRet; @@ -701,7 +703,7 @@ finalize_it: * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) { DEFiRet; @@ -745,7 +747,7 @@ finalize_it: * * rgerhards, 2008-01-10 */ -rsRetVal strmRecordBegin(strm_t *pThis) +static rsRetVal strmRecordBegin(strm_t *pThis) { ASSERT(pThis != NULL); ASSERT(pThis->bInRecord == 0); @@ -753,7 +755,7 @@ rsRetVal strmRecordBegin(strm_t *pThis) return RS_RET_OK; } -rsRetVal strmRecordEnd(strm_t *pThis) +static rsRetVal strmRecordEnd(strm_t *pThis) { DEFiRet; ASSERT(pThis != NULL); @@ -775,7 +777,7 @@ rsRetVal strmRecordEnd(strm_t *pThis) * We do not serialize the dynamic properties. * rgerhards, 2008-01-10 */ -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) +static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) { DEFiRet; int i; @@ -821,7 +823,7 @@ finalize_it: * any new set overwrites the previous one. * rgerhards, 2008-02-27 */ -rsRetVal +static rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt) { DEFiRet; @@ -841,8 +843,8 @@ strmSetWCntr(strm_t *pThis, number_t *pWCnt) /* This function can be used as a generic way to set properties. * rgerhards, 2008-01-11 */ -#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) -rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) +#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, UCHAR_CONSTANT(name), sizeof(name) - 1) +static rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) { DEFiRet; @@ -881,7 +883,7 @@ finalize_it: * reported on the second call may actually be lower than on the first call. This is due to * file circulation. A caller must deal with that. -- rgerhards, 2008-01-30 */ -rsRetVal +static rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs) { DEFiRet; @@ -909,8 +911,33 @@ CODESTARTobjQueryInterface(strm) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - /*xxxpIf->oID = OBJvm; SAMPLE */ - + pIf->Construct = strmConstruct; + pIf->ConstructFinalize = strmConstructFinalize; + pIf->Destruct = strmDestruct; + pIf->ReadChar = strmReadChar; + pIf->UnreadChar = strmUnreadChar; + pIf->ReadLine = strmReadLine; + pIf->SeekCurrOffs = strmSeekCurrOffs; + pIf->Write = strmWrite; + pIf->WriteChar = strmWriteChar; + pIf->WriteLong = strmWriteLong; + pIf->SetFName = strmSetFName; + pIf->SetDir = strmSetDir; + pIf->Flush = strmFlush; + pIf->RecordBegin = strmRecordBegin; + pIf->RecordEnd = strmRecordEnd; + pIf->Serialize = strmSerialize; + pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; + pIf->GetCurrOffset = strmGetCurrOffset; + pIf->SetWCntr = strmSetWCntr; + /* set methods */ + pIf->SetbDeleteOnClose = strmSetbDeleteOnClose; + pIf->SetiMaxFileSize = strmSetiMaxFileSize; + pIf->SetiMaxFiles = strmSetiMaxFiles; + pIf->SetiFileNumDigits = strmSetiFileNumDigits; + pIf->SettOperationsMode = strmSettOperationsMode; + pIf->SettOpenMode = strmSettOpenMode; + pIf->SetsType = strmSetsType; finalize_it: ENDobjQueryInterface(strm) @@ -921,13 +948,10 @@ ENDobjQueryInterface(strm) */ BEGINObjClassInit(strm, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ - OBJSetMethodHandler(objMethod_SERIALIZE, strmSerialize); OBJSetMethodHandler(objMethod_SETPROPERTY, strmSetProperty); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmConstructFinalize); ENDObjClassInit(strm) - -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stream.h b/runtime/stream.h index 371358ab..ece33270 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -93,39 +93,39 @@ typedef struct strm_s { /* interfaces */ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(strm_t **ppThis); + rsRetVal (*ConstructFinalize)(strm_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strm_t **ppThis); + rsRetVal (*SetMaxFileSize)(strm_t *pThis, int64 iMaxFileSize); + rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); + rsRetVal (*ReadChar)(strm_t *pThis, uchar *pC); + rsRetVal (*UnreadChar)(strm_t *pThis, uchar c); + rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr); + rsRetVal (*SeekCurrOffs)(strm_t *pThis); + rsRetVal (*Write)(strm_t *pThis, uchar *pBuf, size_t lenBuf); + rsRetVal (*WriteChar)(strm_t *pThis, uchar c); + rsRetVal (*WriteLong)(strm_t *pThis, long i); + rsRetVal (*SetFName)(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); + rsRetVal (*SetDir)(strm_t *pThis, uchar *pszDir, size_t iLenDir); + rsRetVal (*Flush)(strm_t *pThis); + rsRetVal (*RecordBegin)(strm_t *pThis); + rsRetVal (*RecordEnd)(strm_t *pThis); + rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm); + rsRetVal (*SetiAddtlOpenFlags)(strm_t *pThis, int iNewVal); + rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); + rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); + INTERFACEpropSetMeth(strm, bDeleteOnClose, int); + INTERFACEpropSetMeth(strm, iMaxFileSize, int); + INTERFACEpropSetMeth(strm, iMaxFiles, int); + INTERFACEpropSetMeth(strm, iFileNumDigits, int); + INTERFACEpropSetMeth(strm, tOperationsMode, int); + INTERFACEpropSetMeth(strm, tOpenMode, mode_t); + INTERFACEpropSetMeth(strm, sType, strmType_t); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ /* prototypes */ -rsRetVal strmConstruct(strm_t **ppThis); -rsRetVal strmConstructFinalize(strm_t __attribute__((unused)) *pThis); -rsRetVal strmDestruct(strm_t **ppThis); -rsRetVal strmSetMaxFileSize(strm_t *pThis, int64 iMaxFileSize); -rsRetVal strmSetFileName(strm_t *pThis, uchar *pszName, size_t iLenName); -rsRetVal strmReadChar(strm_t *pThis, uchar *pC); -rsRetVal strmUnreadChar(strm_t *pThis, uchar c); -rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr); -rsRetVal strmSeekCurrOffs(strm_t *pThis); -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); -rsRetVal strmWriteChar(strm_t *pThis, uchar c); -rsRetVal strmWriteLong(strm_t *pThis, long i); -rsRetVal strmSetFName(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); -rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir); -rsRetVal strmFlush(strm_t *pThis); -rsRetVal strmRecordBegin(strm_t *pThis); -rsRetVal strmRecordEnd(strm_t *pThis); -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm); -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal); -rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs); -rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt); PROTOTYPEObjClassInit(strm); -PROTOTYPEpropSetMeth(strm, bDeleteOnClose, int); -PROTOTYPEpropSetMeth(strm, iMaxFileSize, int); -PROTOTYPEpropSetMeth(strm, iMaxFiles, int); -PROTOTYPEpropSetMeth(strm, iFileNumDigits, int); -PROTOTYPEpropSetMeth(strm, tOperationsMode, int); -PROTOTYPEpropSetMeth(strm, tOpenMode, mode_t); -PROTOTYPEpropSetMeth(strm, sType, strmType_t); #endif /* #ifndef STREAM_H_INCLUDED */ diff --git a/runtime/sysvar.c b/runtime/sysvar.c index c102d1f5..4a6ace19 100644 --- a/runtime/sysvar.c +++ b/runtime/sysvar.c @@ -175,8 +175,6 @@ CODESTARTobjQueryInterface(sysvar) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - //xxxpIf->oID = "sysvar";//OBJsysvar; - pIf->Construct = sysvarConstruct; pIf->ConstructFinalize = sysvarConstructFinalize; pIf->Destruct = sysvarDestruct; diff --git a/tools/omfile.c b/tools/omfile.c index d1274f1b..689bad0c 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -776,7 +776,7 @@ doFlush(instanceData *pData) if(pData->poBuf->iBuf == 0) FINALIZE; /* nothing to write, but make this a valid case */ - if(1) { // zlib enabled! + if(0) { // zlib enabled! CHKiRet(doZipWrite(pData)); } else { CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); diff --git a/tools/syslogd.c b/tools/syslogd.c index b43b7a37..22cb6879 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -136,7 +136,6 @@ #include "errmsg.h" #include "datetime.h" #include "parser.h" -#include "sysvar.h" #include "unicode-helper.h" /* definitions for objects we access */ @@ -3340,26 +3339,6 @@ GlobalClassExit(void) objRelease(datetime, CORE_COMPONENT); /* TODO: implement the rest of the deinit */ -#if 0 - CHKiRet(datetimeClassInit(NULL)); - CHKiRet(msgClassInit(NULL)); - CHKiRet(strmClassInit(NULL)); - CHKiRet(wtiClassInit(NULL)); - CHKiRet(wtpClassInit(NULL)); - CHKiRet(qqueueClassInit(NULL)); - CHKiRet(vmstkClassInit(NULL)); - CHKiRet(sysvarClassInit(NULL)); - CHKiRet(vmClassInit(NULL)); - CHKiRet(vmopClassInit(NULL)); - CHKiRet(vmprgClassInit(NULL)); - CHKiRet(ctok_tokenClassInit(NULL)); - CHKiRet(ctokClassInit(NULL)); - CHKiRet(exprClassInit(NULL)); - - /* dummy "classes" */ - CHKiRet(actionClassInit()); - CHKiRet(templateInit()); -#endif /* dummy "classes */ strExit(); -- cgit From f2800ba261d2fb7466cbdebbf80afe92f0bffd3d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 15:10:24 +0200 Subject: modified stream class and omfile to work with it now some basic operations are carried out via the stream class. --- doc/rsyslog_conf_global.html | 3 + runtime/queue.c | 3 +- runtime/stream.c | 168 ++++++++++++++++++++++++++++++++++++------- runtime/stream.h | 11 ++- tools/omfile.c | 94 +++++++++++++++++------- 5 files changed, 225 insertions(+), 54 deletions(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 778e18f8..0f408fcf 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -189,6 +189,9 @@ supported in order to be compliant to the upcoming new syslog RFC series.
  • $MaxOpenFiles
  • $ModDir
  • $ModLoad
    • +
  • $OMFileZipLevel 0..9 [default 0] - if greater 0, turns on gzip compression +of the output file. The higher the number, the better the compression, but also the +more CPU is required for zipping.
  • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/runtime/queue.c b/runtime/queue.c index 3f14b535..3532a145 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1923,8 +1923,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) } CHKiRet(strm.Construct(&psQIF)); - CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE)); - CHKiRet(strm.SetiAddtlOpenFlags(psQIF, O_TRUNC)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); CHKiRet(strm.ConstructFinalize(psQIF)); diff --git a/runtime/stream.c b/runtime/stream.c index 261eb9ca..abcfce0c 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1,4 +1,3 @@ -//TODO: O_TRUC mode! /* The serial stream class. * * A serial stream provides serial data access. In theory, serial streams @@ -50,6 +49,7 @@ /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(zlibw) /* forward definitions */ static rsRetVal strmFlush(strm_t *pThis); @@ -74,7 +74,6 @@ static rsRetVal strmOpenFile(strm_t *pThis) int iFlags; ASSERT(pThis != NULL); - ASSERT(pThis->tOperationsMode == STREAMMODE_READ || pThis->tOperationsMode == STREAMMODE_WRITE); if(pThis->fd != -1) ABORT_FINALIZE(RS_RET_OK); @@ -95,13 +94,27 @@ static rsRetVal strmOpenFile(strm_t *pThis) } } +RUNLOG_VAR("%d", pThis->tOperationsMode); /* compute which flags we need to provide to open */ - if(pThis->tOperationsMode == STREAMMODE_READ) - iFlags = O_RDONLY; - else - iFlags = O_WRONLY | O_CREAT; + switch(pThis->tOperationsMode) { + case STREAMMODE_READ: + iFlags = O_CLOEXEC | O_NOCTTY | O_RDONLY; + break; + case STREAMMODE_WRITE: /* legacy mode used inside queue engine */ + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT; + break; + case STREAMMODE_WRITE_TRUNC: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_TRUNC; + break; + case STREAMMODE_WRITE_APPEND: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_APPEND; + break; + default:assert(0); + break; + } - iFlags |= pThis->iAddtlOpenFlags; + iFlags |= pThis->iAddtlOpenFlags; // TODO: remove this! +dbgprintf("XXX: open with flags %d\n", iFlags); pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); if(pThis->fd == -1) { @@ -135,7 +148,7 @@ static rsRetVal strmCloseFile(strm_t *pThis) ASSERT(pThis->fd != -1); dbgoprint((obj_t*) pThis, "file %d closing\n", pThis->fd); - if(pThis->tOperationsMode == STREAMMODE_WRITE) + if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); close(pThis->fd); // TODO: error check @@ -398,14 +411,25 @@ ENDobjConstruct(strm) */ static rsRetVal strmConstructFinalize(strm_t *pThis) { + rsRetVal localRet; DEFiRet; ASSERT(pThis != NULL); - if(pThis->pIOBuf == NULL) { /* allocate our io buffer in case we have not yet */ - if((pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - pThis->iBufPtrMax = 0; /* results in immediate read request */ + CHKmalloc(pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + pThis->iBufPtrMax = 0; /* results in immediate read request */ + if(pThis->iZipLevel) { /* do we need a zip buf? */ + localRet = objUse(zlibw, LM_ZLIBW_FILENAME); + if(localRet != RS_RET_OK) { + pThis->iZipLevel = 0; + DBGPRINTF("stream was requested with zip mode, but zlibw module unavailable (%d) - using " + "without zip\n", localRet); + } else { + /* we use the same size as the original buf, as we would like + * to make sure we can write out everyting with a SINGLE api call! + */ + CHKmalloc(pThis->pZipBuf = (Bytef*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + } } finalize_it: @@ -416,15 +440,20 @@ finalize_it: /* destructor for the strm object */ BEGINobjDestruct(strm) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(strm) - if(pThis->tOperationsMode == STREAMMODE_WRITE) + if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); /* ... then free resources */ if(pThis->fd != -1) strmCloseFile(pThis); + if(pThis->iZipLevel) { /* do we need a zip buf? */ + objRelease(zlibw, LM_ZLIBW_FILENAME); + } + free(pThis->pszDir); free(pThis->pIOBuf); + free(pThis->pZipBuf); free(pThis->pszCurrFName); free(pThis->pszFName); ENDobjDestruct(strm) @@ -453,20 +482,17 @@ finalize_it: } -/* write memory buffer to a stream object. - * To support direct writes of large objects, this method may be called - * with a buffer pointing to some region other than the stream buffer itself. - * However, in that case the stream buffer must be empty (strmFlush() has to - * be called before), because we would otherwise mess up with the sequence - * inside the stream. -- rgerhards, 2008-01-10 +/* physically write to the output file. the provided data is ready for + * writing (e.g. zipped if we are requested to do that). + * rgerhards, 2009-06-04 */ -static rsRetVal strmWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static rsRetVal +strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; int iWritten; ASSERT(pThis != NULL); - ASSERT(pBuf == pThis->pIOBuf || pThis->iBufPtr == 0); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); @@ -499,6 +525,95 @@ finalize_it: } +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + * Note that we always do a full deflateInit ... deflate ... deflateEnd + * sequence. While this is not optimal, we need to do it because we need + * to ensure that the file is readable even when we are aborted. Doing the + * full sequence brings us as far towards this goal as possible (and not + * doing it would be a total failure). It may be worth considering to + * add a config switch so that the user can decide the risk he is ready + * to take, but so far this is not yet implemented (not even requested ;)). + * rgerhards, 2009-06-04 + */ +static rsRetVal +doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + z_stream zstrm; + int zRet; /* zlib return state */ + DEFiRet; + assert(pThis != NULL); + assert(pBuf != NULL); + + /* allocate deflate state */ + zstrm.zalloc = Z_NULL; + zstrm.zfree = Z_NULL; + zstrm.opaque = Z_NULL; + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateInit2() done successfully\n"); + + /* now doing the compression */ + zstrm.avail_in = lenBuf; + zstrm.next_in = (Bytef*) pBuf; + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); + zstrm.avail_out = pThis->sIOBufSize; + zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, pThis->sIOBufSize - zstrm.avail_out)); + } while (zstrm.avail_out == 0); + assert(zstrm.avail_in == 0); /* all input will be used */ + +RUNLOG_STR("deflate() should be done successfully\n"); + + zRet = zlibw.DeflateEnd(&zstrm); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateEnd() done successfully\n"); + +finalize_it: + RETiRet; +} + + +/* write memory buffer to a stream object. + * To support direct writes of large objects, this method may be called + * with a buffer pointing to some region other than the stream buffer itself. + * However, in that case the stream buffer must be empty (strmFlush() has to + * be called before), because we would otherwise mess up with the sequence + * inside the stream. -- rgerhards, 2008-01-10 + */ +static rsRetVal +strmWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + DEFiRet; + + ASSERT(pThis != NULL); + ASSERT(pBuf == pThis->pIOBuf || pThis->iBufPtr == 0); + + if(pThis->iZipLevel) { + CHKiRet(doZipWrite(pThis, pBuf, lenBuf)); + } else { + /* write without zipping */ + CHKiRet(strmPhysWrite(pThis, pBuf, lenBuf)); + } + +finalize_it: + RETiRet; +} + + /* flush stream output buffer to persistent storage. This can be called at any time * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 @@ -511,7 +626,7 @@ strmFlush(strm_t *pThis) ASSERT(pThis != NULL); dbgoprint((obj_t*) pThis, "file %d flush, buflen %ld\n", pThis->fd, (long) pThis->iBufPtr); - if(pThis->tOperationsMode == STREAMMODE_WRITE && pThis->iBufPtr > 0) { + if(pThis->tOperationsMode != STREAMMODE_READ && pThis->iBufPtr > 0) { iRet = strmWriteInternal(pThis, pThis->pIOBuf, pThis->iBufPtr); } @@ -605,6 +720,7 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) ASSERT(pThis != NULL); ASSERT(pBuf != NULL); +dbgprintf("strmWrite(%p, '%s', %ld);\n", pThis, pBuf,lenBuf); /* check if the to-be-written data is larger than our buffer size */ if(lenBuf >= pThis->sIOBufSize) { /* it is - so we do a direct write, that is most efficient. @@ -646,6 +762,7 @@ DEFpropSetMeth(strm, iFileNumDigits, int) DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) +DEFpropSetMeth(strm, iZipLevel, int) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { @@ -681,13 +798,14 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) ASSERT(pThis != NULL); ASSERT(pszName != NULL); +dbgprintf("XXX: strm setFname: '%s'\n", pszName); if(iLenName < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); if(pThis->pszFName != NULL) free(pThis->pszFName); - if((pThis->pszFName = malloc(sizeof(uchar) * iLenName + 1)) == NULL) + if((pThis->pszFName = malloc(sizeof(uchar) * (iLenName + 1))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */ @@ -711,6 +829,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) ASSERT(pThis != NULL); ASSERT(pszDir != NULL); +dbgprintf("XXX: strm setDir: '%s'\n", pszDir); if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); @@ -927,7 +1046,7 @@ CODESTARTobjQueryInterface(strm) pIf->RecordBegin = strmRecordBegin; pIf->RecordEnd = strmRecordEnd; pIf->Serialize = strmSerialize; - pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; + /* TODO: remove this! */ pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; pIf->GetCurrOffset = strmGetCurrOffset; pIf->SetWCntr = strmSetWCntr; /* set methods */ @@ -938,6 +1057,7 @@ CODESTARTobjQueryInterface(strm) pIf->SettOperationsMode = strmSettOperationsMode; pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; + pIf->SetiZipLevel = strmSetiZipLevel; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index ece33270..449bf6c6 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -47,6 +47,7 @@ #include "obj-types.h" #include "glbl.h" #include "stream.h" +#include "zlibw.h" /* stream types */ typedef enum { @@ -55,10 +56,12 @@ typedef enum { STREAMTYPE_FILE_MONITOR = 2 /**< monitor a (third-party) file */ } strmType_t; -typedef enum { +typedef enum { /* when extending, do NOT change existing modes! */ STREAMMMODE_INVALID = 0, STREAMMODE_READ = 1, - STREAMMODE_WRITE = 2 + STREAMMODE_WRITE = 2, + STREAMMODE_WRITE_TRUNC = 3, + STREAMMODE_WRITE_APPEND = 4 } strmMode_t; /* The strm_t data structure */ @@ -89,6 +92,9 @@ typedef struct strm_s { size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ + Bytef *pZipBuf; + } strm_t; /* interfaces */ @@ -121,6 +127,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, tOperationsMode, int); INTERFACEpropSetMeth(strm, tOpenMode, mode_t); INTERFACEpropSetMeth(strm, sType, strmType_t); + INTERFACEpropSetMeth(strm, iZipLevel, int); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tools/omfile.c b/tools/omfile.c index 689bad0c..3f517906 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -70,6 +70,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,7 @@ #include "module-template.h" #include "errmsg.h" #include "unicode-helper.h" +#include "stream.h" #include "zlibw.h" MODULE_TYPE_OUTPUT @@ -96,6 +98,7 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(zlibw) +DEFobjCurrIf(strm) /* The following structure is a dynafile name cache entry. */ @@ -131,12 +134,14 @@ static uid_t dirUID; /* UID to be used for newly created directories */ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ +static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ + strm_t *pStrm; /* our output stream */ short fd; /* file descriptor for (current) file */ outbuf_t *poBuf; /* output buffer */ enum { @@ -165,6 +170,7 @@ typedef struct _instanceData { dynaFileCacheEntry **dynCache; off_t f_sizeLimit; /* file size limit, 0 = no limit */ uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + int iZipLevel; /* zip mode to use for this selector */ } instanceData; @@ -446,13 +452,11 @@ prepareFile(instanceData *pData, uchar *newFileName) FINALIZE; /* we are done in this case */ } - if(access((char*)newFileName, F_OK) == 0) { - /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - } else { - pData->fd = -1; + // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 + + if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ + pData->fd = -1; if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code @@ -485,8 +489,29 @@ prepareFile(instanceData *pData, uchar *newFileName) */ } } + close(pData->fd); /* close again, as we need a stream further on */ } } + + char szNameBuf[MAXFNAME]; + char szDirName[MAXFNAME]; + char szBaseName[MAXFNAME]; + strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szDirName, dirname(szNameBuf)); + strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szBaseName, basename(szNameBuf)); +DBGPRINTF("XXX: name to set: '%s', dirname '%s'\n", pData->f_fname, szDirName); + + CHKiRet(strm.Construct(&pData->pStrm)); + CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); + CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); + CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); + CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); + CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.ConstructFinalize(pData->pStrm)); + + pData->fd = 2; /* TODO: dummy to keep current inconsistent code happy - remove later */ + finalize_it: /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean * non-open file descriptor. Anyhow, I leave this comment for the time being to that if @@ -715,20 +740,20 @@ static rsRetVal doZipWrite(instanceData *pData) { outbuf_t *poBuf; - z_stream strm; + z_stream zstrm; int zRet; /* zlib return state */ DEFiRet; assert(pData != NULL); poBuf = pData->poBuf; /* use as a shortcut */ - strm = poBuf->zStrm; /* another shortcut */ + zstrm = poBuf->zStrm; /* another shortcut */ /* allocate deflate state */ - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; + zstrm.zalloc = Z_NULL; + zstrm.zfree = Z_NULL; + zstrm.opaque = Z_NULL; /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&strm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + zRet = zlibw.DeflateInit2(&zstrm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); @@ -736,24 +761,24 @@ doZipWrite(instanceData *pData) RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ - strm.avail_in = poBuf->iBuf; - strm.next_in = (Bytef*) poBuf->pszBuf; + zstrm.avail_in = poBuf->iBuf; + zstrm.next_in = (Bytef*) poBuf->pszBuf; /* run deflate() on input until output buffer not full, finish compression if all of source has been read in */ do { - dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", strm.avail_in, strm.total_in); - strm.avail_out = OUTBUF_LEN; - strm.next_out = (Bytef*) poBuf->zipBuf; - zRet = zlibw.Deflate(&strm, Z_FINISH); /* no bad return value */ - dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, strm.avail_out); + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); + zstrm.avail_out = OUTBUF_LEN; + zstrm.next_out = (Bytef*) poBuf->zipBuf; + zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - strm.avail_out)); - } while (strm.avail_out == 0); - assert(strm.avail_in == 0); /* all input will be used */ + CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - zstrm.avail_out)); + } while (zstrm.avail_out == 0); + assert(zstrm.avail_in == 0); /* all input will be used */ RUNLOG_STR("deflate() should be done successfully\n"); - zRet = zlibw.DeflateEnd(&strm); + zRet = zlibw.DeflateEnd(&zstrm); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); @@ -802,8 +827,14 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pszBuf != NULL); poBuf = pData->poBuf; /* use as a shortcut */ -dbgprintf("doWrite, pData->fd %d, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", -pData->fd, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); +dbgprintf("doWrite, pData->fd %d, pData->pStrm %p, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", +pData->fd, pData->pStrm, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); + +RUNLOG_VAR("%p", pData->pStrm); + if(pData->pStrm != NULL){ + CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); + FINALIZE; // TODO: clean up later + } if(pData->fd != poBuf->fd) { // TODO: more efficient use for dynafiles @@ -869,6 +900,10 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ + if(pData->pStrm != NULL) { +RUNLOG_STR("XXX: destructing stream"); + strm.Destruct(&pData->pStrm); + } if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->fd != -1) @@ -945,6 +980,7 @@ CODESTARTparseSelectorAct pData->fileGID = fileGID; pData->dirUID = dirUID; pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */ /* we now allocate the cache table. We use calloc() intentionally, as we * need all pointers to be initialized to NULL pointers. @@ -983,6 +1019,7 @@ CODESTARTparseSelectorAct pData->fileGID = fileGID; pData->dirUID = dirUID; pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; /* at this stage, we ignore the return value of prepareFile, this is taken * care of in later steps. -- rgerhards, 2009-03-19 @@ -1021,6 +1058,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a fDirCreateMode = 0700; bCreateDirs = 1; bEnableSync = 0; + iZipLevel = 0; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -1046,6 +1084,8 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); + objRelease(strm, CORE_COMPONENT); objRelease(zlibw, LM_ZLIBW_FILENAME); free(pszTplName); ENDmodExit @@ -1062,9 +1102,11 @@ BEGINmodInit(File) CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr - CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); + CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID)); -- cgit From 76da7f9f4e3dc900046a1956153d10532f2b1ae0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 15:59:37 +0200 Subject: added $OMFileIOBufferSize config directive and plumbing --- ChangeLog | 3 ++ doc/rsyslog_conf_global.html | 1 + plugins/imfile/imfile.c | 3 +- runtime/stream.c | 21 +------- runtime/stream.h | 3 +- runtime/unicode-helper.h | 5 ++ tools/omfile.c | 114 ++++++++++++++++++++----------------------- tools/syslogd.c | 8 +-- 8 files changed, 69 insertions(+), 89 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9d398f09..8fc4194d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,9 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - reduced max value for $DynaFileCacheSize to 1,000 (the former maximum of 10,000 really made no sense, even 1,000 is very high, but we like to keep the user in control ;)). +- added configuration commands (see doc for explanations) + * $OMFileZipLevel + * $OMFileIOBufferSize --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 0f408fcf..1e268f4b 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -192,6 +192,7 @@ supported in order to be compliant to the upcoming new syslog RFC series.
  • $OMFileZipLevel 0..9 [default 0] - if greater 0, turns on gzip compression of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.
    • +
  • $OMFileIOBufferSize <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)
  • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index ff32c857..42c70539 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -357,8 +357,7 @@ persistStrmState(fileInfo_t *pInfo) /* TODO: create a function persistObj in obj.c? */ CHKiRet(strm.Construct(&psSF)); CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE)); - CHKiRet(strm.SetiAddtlOpenFlags(psSF, O_TRUNC)); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); CHKiRet(strm.ConstructFinalize(psSF)); diff --git a/runtime/stream.c b/runtime/stream.c index abcfce0c..a4844d2b 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -94,7 +94,6 @@ static rsRetVal strmOpenFile(strm_t *pThis) } } -RUNLOG_VAR("%d", pThis->tOperationsMode); /* compute which flags we need to provide to open */ switch(pThis->tOperationsMode) { case STREAMMODE_READ: @@ -113,9 +112,6 @@ RUNLOG_VAR("%d", pThis->tOperationsMode); break; } - iFlags |= pThis->iAddtlOpenFlags; // TODO: remove this! -dbgprintf("XXX: open with flags %d\n", iFlags); - pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); if(pThis->fd == -1) { int ierrnoSave = errno; @@ -763,6 +759,7 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, sIOBufSize, size_t) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { @@ -771,19 +768,6 @@ static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) return RS_RET_OK; } -static rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) -{ - DEFiRet; - - if(iNewVal & O_APPEND) - ABORT_FINALIZE(RS_RET_PARAM_ERROR); - - pThis->iAddtlOpenFlags = iNewVal; - -finalize_it: - RETiRet; -} - /* set the stream's file prefix * The passed-in string is duplicated. So if the caller does not need @@ -829,7 +813,6 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) ASSERT(pThis != NULL); ASSERT(pszDir != NULL); -dbgprintf("XXX: strm setDir: '%s'\n", pszDir); if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); @@ -1046,7 +1029,6 @@ CODESTARTobjQueryInterface(strm) pIf->RecordBegin = strmRecordBegin; pIf->RecordEnd = strmRecordEnd; pIf->Serialize = strmSerialize; - /* TODO: remove this! */ pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; pIf->GetCurrOffset = strmGetCurrOffset; pIf->SetWCntr = strmSetWCntr; /* set methods */ @@ -1058,6 +1040,7 @@ CODESTARTobjQueryInterface(strm) pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetsIOBufSize = strmSetsIOBufSize; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index 449bf6c6..e4871611 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -74,7 +74,6 @@ typedef struct strm_s { int lenFName; strmMode_t tOperationsMode; mode_t tOpenMode; - int iAddtlOpenFlags; /* can be used to specifiy additional (compatible!) open flags */ int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ @@ -117,7 +116,6 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*RecordBegin)(strm_t *pThis); rsRetVal (*RecordEnd)(strm_t *pThis); rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm); - rsRetVal (*SetiAddtlOpenFlags)(strm_t *pThis, int iNewVal); rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); INTERFACEpropSetMeth(strm, bDeleteOnClose, int); @@ -128,6 +126,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, tOpenMode, mode_t); INTERFACEpropSetMeth(strm, sType, strmType_t); INTERFACEpropSetMeth(strm, iZipLevel, int); + INTERFACEpropSetMeth(strm, sIOBufSize, size_t); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index fe4cc1f8..5289da3f 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -31,6 +31,11 @@ #include +static inline char* ustrncpy(uchar *psz1, uchar *psz2, size_t len) +{ + return strncpy((char*) psz1, (char*) psz2, len); +} + static inline int ustrcmp(uchar *psz1, uchar *psz2) { return strcmp((char*) psz1, (char*) psz2); diff --git a/tools/omfile.c b/tools/omfile.c index 3f517906..85f04f13 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -123,6 +123,8 @@ typedef struct { } outbuf_t; +#define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ + /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ static int fCreateMode = 0644; /* mode to use when creating files */ @@ -135,6 +137,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ +static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -168,9 +171,10 @@ typedef struct _instanceData { * pointer points to the overall structure. */ dynaFileCacheEntry **dynCache; - off_t f_sizeLimit; /* file size limit, 0 = no limit */ - uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *iSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ + int iIOBufSize; /* size of associated io buffer */ } instanceData; @@ -293,12 +297,12 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR } /* OK, we finally got a correct template. So let's use it... */ - strncpy((char*) pData->f_fname, (char*) pOch->pszFileTemplate, MAXFNAME); - pData->f_sizeLimit = pOch->uSizeLimit; + ustrncpy(pData->f_fname, pOch->pszFileTemplate, MAXFNAME); + pData->iSizeLimit = pOch->uSizeLimit; /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->f_sizeLimitCmd = pOch->cmdOnSizeLimit; + pData->iSizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -322,7 +326,7 @@ int resolveFileSizeLimit(instanceData *pData) off_t actualFileSize; ASSERT(pData != NULL); - if(pData->f_sizeLimitCmd == NULL) + if(pData->iSizeLimitCmd == NULL) return 1; /* nothing we can do in this case... */ /* the execProg() below is probably not great, but at least is is @@ -334,7 +338,7 @@ int resolveFileSizeLimit(instanceData *pData) * when we have a space in the program name. If we find it, everything after * the space is treated as a single argument. */ - if((pCmd = ustrdup(pData->f_sizeLimitCmd)) == NULL) { + if((pCmd = ustrdup(pData->iSizeLimitCmd)) == NULL) { /* there is not much we can do - we make syslogd close the file in this case */ return 1; } @@ -357,7 +361,7 @@ int resolveFileSizeLimit(instanceData *pData) pData->fCreateMode); actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { + if(actualFileSize >= pData->iSizeLimit) { /* OK, it didn't work out... */ return 1; } @@ -500,12 +504,14 @@ prepareFile(instanceData *pData, uchar *newFileName) strcpy(szDirName, dirname(szNameBuf)); strcpy(szNameBuf, (char*)pData->f_fname); strcpy(szBaseName, basename(szNameBuf)); -DBGPRINTF("XXX: name to set: '%s', dirname '%s'\n", pData->f_fname, szDirName); CHKiRet(strm.Construct(&pData->pStrm)); CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); +dbgprintf("static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); +RUNLOG_VAR("%d", pData->iIOBufSize); + CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); @@ -650,9 +656,9 @@ again: /* check if we have a file size limit and, if so, * obey to it. */ - if(pData->f_sizeLimit != 0) { + if(pData->iSizeLimit != 0) { actualFileSize = lseek(fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { + if(actualFileSize >= pData->iSizeLimit) { char errMsg[256]; /* for now, we simply disable a file once it is * beyond the maximum size. This is better than having @@ -665,14 +671,14 @@ again: /* didn't work out, so disable... */ snprintf(errMsg, sizeof(errMsg), "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); + pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); errno = 0; errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg); ABORT_FINALIZE(RS_RET_DISABLE_ACTION); } else { snprintf(errMsg, sizeof(errMsg), "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); + pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); errno = 0; errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); } @@ -901,7 +907,6 @@ BEGINfreeInstance CODESTARTfreeInstance doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->pStrm != NULL) { -RUNLOG_STR("XXX: destructing stream"); strm.Destruct(&pData->pStrm); } if(pData->bDynamicName) { @@ -934,12 +939,11 @@ CODESTARTparseSelectorAct pData->bSyncFile = 0; p++; } else { - pData->bSyncFile = bEnableSync ? 1 : 0; + pData->bSyncFile = bEnableSync; } - pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */ + pData->iSizeLimit = 0; /* default value, use outchannels to configure! */ - switch (*p) - { + switch(*p) { case '$': CODE_STD_STRING_REQUESTparseSelectorAct(1) /* rgerhards 2005-06-21: this is a special setting for output-channel @@ -949,13 +953,8 @@ CODESTARTparseSelectorAct * rgerhards, 2007-07-24: output-channels will go away. We keep them * for compatibility reasons, but seems to have been a bad idea. */ - if((iRet = cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)) == RS_RET_OK) { - pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - } + CHKiRet(cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)); + pData->bDynamicName = 0; break; case '?': /* This is much like a regular file handle, but we need to obtain @@ -972,16 +971,6 @@ CODESTARTparseSelectorAct pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ - pData->fCreateMode = fCreateMode; /* freeze current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - pData->iZipLevel = iZipLevel; - pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */ /* we now allocate the cache table. We use calloc() intentionally, as we * need all pointers to be initialized to NULL pointers. */ @@ -995,50 +984,49 @@ CODESTARTparseSelectorAct case '|': case '/': CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* rgerhards, 2007-0726: first check if file or pipe */ if(*p == '|') { pData->fileType = eTypePIPE; ++p; } else { pData->fileType = eTypeFILE; } - /* rgerhards 2004-11-17: from now, we need to have different - * processing, because after the first comma, the template name - * to use is specified. So we need to scan for the first coma first - * and then look at the rest of the line. - */ CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); - pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - pData->iZipLevel = iZipLevel; - - /* at this stage, we ignore the return value of prepareFile, this is taken - * care of in later steps. -- rgerhards, 2009-03-19 + break; + default: + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* freeze current paremeters for this action */ + pData->iDynaFileCacheSize = iDynaFileCacheSize; + pData->fCreateMode = fCreateMode; + pData->fDirCreateMode = fDirCreateMode; + pData->bCreateDirs = bCreateDirs; + pData->bFailOnChown = bFailOnChown; + pData->fileUID = fileUID; + pData->fileGID = fileGID; + pData->dirUID = dirUID; + pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; + pData->iIOBufSize = iIOBufSize; + + if(pData->bDynamicName == 0) { + if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) + pData->fileType = eTypeCONSOLE; + /* try open and emit error message if not possible. At this stage, we ignore the + * return value of prepareFile, this is taken care of in later steps. */ prepareFile(pData, pData->f_fname); if(pData->fd < 0 ) { - pData->fd = -1; DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); - break; } - if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) - pData->fileType = eTypeCONSOLE; - break; - default: - iRet = RS_RET_CONFLINE_UNPROCESSED; - break; } + +dbgprintf("in init static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); +RUNLOG_VAR("%d", pData->iIOBufSize); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -1059,6 +1047,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; + iIOBufSize = IOBUF_DFLT_SIZE; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -1107,6 +1096,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID)); diff --git a/tools/syslogd.c b/tools/syslogd.c index 22cb6879..206b79e8 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3042,18 +3042,18 @@ static rsRetVal loadBuildInModules(void) { DEFiRet; - if((iRet = module.doModInit(modInitFile, (uchar*) "builtin-file", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitFile, UCHAR_CONSTANT("builtin-file"), NULL)) != RS_RET_OK) { RETiRet; } #ifdef SYSLOG_INET - if((iRet = module.doModInit(modInitFwd, (uchar*) "builtin-fwd", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitFwd, UCHAR_CONSTANT("builtin-fwd"), NULL)) != RS_RET_OK) { RETiRet; } #endif - if((iRet = module.doModInit(modInitShell, (uchar*) "builtin-shell", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitShell, UCHAR_CONSTANT("builtin-shell"), NULL)) != RS_RET_OK) { RETiRet; } - if((iRet = module.doModInit(modInitDiscard, (uchar*) "builtin-discard", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitDiscard, UCHAR_CONSTANT("builtin-discard"), NULL)) != RS_RET_OK) { RETiRet; } -- cgit From 68f0ffb29acf5ff07ccd8f30581d96d6915a834e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Jun 2009 15:41:05 +0200 Subject: made dynafiles working again now only stream class is utilized. ttys, pipes and outchannel functionality is currently disabled. But the testbench worked again. Cleanup needed, will do this with next commit (it may break things and I like to have this milestone here). --- tests/diag.sh | 3 +- tools/omfile.c | 148 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 75 insertions(+), 76 deletions(-) diff --git a/tests/diag.sh b/tests/diag.sh index 1ceca75b..1fa8f62a 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -9,7 +9,7 @@ #valgrind="valgrind --tool=drd --log-fd=1" #valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout printmutexaction" +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" #export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason @@ -17,6 +17,7 @@ case $1 in rm -f rsyslogd.started work-*.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files rm -rf test-spool + rm -f core.* vgcore.* mkdir test-spool ;; 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf diff --git a/tools/omfile.c b/tools/omfile.c index 85f04f13..dd585893 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -104,7 +104,7 @@ DEFobjCurrIf(strm) */ struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ - short fd; /* name associated with file name in cache */ + strm_t *pStrm; /* our output stream */ time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; @@ -116,7 +116,6 @@ typedef struct { uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ size_t lenBuf; /* max size of buffer */ size_t iBuf; /* current buffer index */ - int fd; /* which file descriptor is this buf for? */ /* elements for zip writing */ z_stream zStrm; char zipBuf[OUTBUF_LEN]; @@ -145,8 +144,7 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - short fd; /* file descriptor for (current) file */ - outbuf_t *poBuf; /* output buffer */ + //outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -203,7 +201,7 @@ CODESTARTdbgPrintInstInfo ); } else { /* regular file */ dbgprintf("%s", pData->f_fname); - if (pData->fd == -1) + if (pData->pStrm == NULL) dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -312,6 +310,7 @@ finalize_it: } +#if 0 /* rgerhards 2005-06-21: Try to resolve a size limit * situation. This first runs the command, and then * checks if we are still above the treshold. @@ -368,6 +367,7 @@ int resolveFileSizeLimit(instanceData *pData) return 0; } +#endif /* This function deletes an entry from the dynamic file name @@ -377,13 +377,12 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void +static rsRetVal dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { + DEFiRet; ASSERT(pCache != NULL); - BEGINfunc; - if(pCache[iEntry] == NULL) FINALIZE; @@ -394,7 +393,8 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) * not the name to be freed. */ if(pCache[iEntry]->pName != NULL) { - close(pCache[iEntry]->fd); + if(pCache[iEntry]->pStrm != NULL) + strm.Destruct(&pCache[iEntry]->pStrm); d_free(pCache[iEntry]->pName); pCache[iEntry]->pName = NULL; } @@ -405,7 +405,7 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) } finalize_it: - ENDfunc; + RETiRet; } @@ -450,17 +450,20 @@ static void dynaFileFreeCache(instanceData *pData) static rsRetVal prepareFile(instanceData *pData, uchar *newFileName) { + int fd; DEFiRet; +#if 0 if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); + fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); FINALIZE; /* we are done in this case */ } +#endif // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ - pData->fd = -1; + fd = -1; if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code @@ -475,17 +478,17 @@ prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, + fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); - if(pData->fd != -1) { + if(fd != -1) { /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { + if(fchown(fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; - close(pData->fd); - pData->fd = -1; + close(fd); + fd = -1; errno = eSave; } /* we will silently ignore the chown() failure @@ -493,16 +496,16 @@ prepareFile(instanceData *pData, uchar *newFileName) */ } } - close(pData->fd); /* close again, as we need a stream further on */ + close(fd); /* close again, as we need a stream further on */ } } char szNameBuf[MAXFNAME]; char szDirName[MAXFNAME]; char szBaseName[MAXFNAME]; - strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szNameBuf, (char*)newFileName); strcpy(szDirName, dirname(szNameBuf)); - strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szNameBuf, (char*)newFileName); strcpy(szBaseName, basename(szNameBuf)); CHKiRet(strm.Construct(&pData->pStrm)); @@ -516,18 +519,18 @@ RUNLOG_VAR("%d", pData->iIOBufSize); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); - pData->fd = 2; /* TODO: dummy to keep current inconsistent code happy - remove later */ - finalize_it: - /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean + /* this was "fd != 0", which I think was a bug. I guess 0 was intended to mean * non-open file descriptor. Anyhow, I leave this comment for the time being to that if * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 */ - if(pData->fd != -1 && isatty(pData->fd)) { - DBGPRINTF("file %d is a tty file\n", pData->fd); +#if 0 // TODO: this must be done by stream class! + if(fd != -1 && isatty(fd)) { + DBGPRINTF("file %d is a tty file\n", fd); pData->fileType = eTypeTTY; untty(); } +#endif RETiRet; } @@ -541,15 +544,16 @@ finalize_it: * be written. * This is a helper to writeFile(). rgerhards, 2007-07-03 */ -static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) +static inline rsRetVal +prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) { time_t ttOldest; /* timestamp of oldest element */ int iOldest; int i; int iFirstFree; + rsRetVal localRet; dynaFileCacheEntry **pCache; - - BEGINfunc + DEFiRet; ASSERT(pData != NULL); ASSERT(newFileName != NULL); @@ -562,8 +566,9 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if( (pData->iCurrElt != -1) && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ - pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ // TODO: optimize time call! + // LRU needs only a strictly monotonically increasing counter, so such a one could do + FINALIZE; } /* ok, no luck. Now let's search the table if we find a matching spot. @@ -579,10 +584,10 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } else { /* got an element, let's see if it matches */ if(!ustrcmp(newFileName, pCache[i]->pName)) { /* we found our element! */ - pData->fd = pCache[i]->fd; + pData->pStrm = pCache[i]->pStrm; pData->iCurrElt = i; pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + FINALIZE; } /* did not find it - so lets keep track of the counters for LRU */ if(pCache[i]->lastUsed < ttOldest) { @@ -598,23 +603,20 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg iFirstFree = pData->iCurrCacheSize++; } + pData->iCurrElt = -1; if(iFirstFree == -1) { dynaFileDelCacheEntry(pCache, iOldest, 0); iFirstFree = iOldest; /* this one *is* now free ;) */ } else { /* we need to allocate memory for the cache structure */ - pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)); - if(pCache[iFirstFree] == NULL) { - DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n"); - return -1; - } + CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry))); } /* Ok, we finally can open the file */ - prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ + localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ /* file is either open now or an error state set */ - if(pData->fd == -1) { + if(pData->pStrm == NULL) { /* do not report anything if the message is an internally-generated * message. Otherwise, we could run into a never-ending loop. The bad * news is that we also lose errors on startup messages, but so it is. @@ -625,22 +627,21 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } dynaFileDelCacheEntry(pCache, iFirstFree, 1); - pData->iCurrElt = -1; - return -1; + ABORT_FINALIZE(localRet); } - pCache[iFirstFree]->fd = pData->fd; - pCache[iFirstFree]->pName = ustrdup(newFileName); /* TODO: check for NULL (very unlikely) */ - pCache[iFirstFree]->lastUsed = time(NULL); + CHKmalloc(pCache[iFirstFree]->pName = ustrdup(newFileName)); + pCache[iFirstFree]->pStrm = pData->pStrm; + pCache[iFirstFree]->lastUsed = time(NULL); // monotonically increasing value! TODO: performance pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); - ENDfunc - - return 0; +finalize_it: + RETiRet; } +#if 0 /* physically write the file */ static rsRetVal @@ -816,6 +817,7 @@ doFlush(instanceData *pData) finalize_it: RETiRet; } +#endif /* do the actual write process. This function is to be called once we are ready for writing. @@ -826,15 +828,12 @@ finalize_it: static rsRetVal doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) { - int i; - outbuf_t *poBuf; DEFiRet; ASSERT(pData != NULL); ASSERT(pszBuf != NULL); - poBuf = pData->poBuf; /* use as a shortcut */ -dbgprintf("doWrite, pData->fd %d, pData->pStrm %p, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", -pData->fd, pData->pStrm, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); +dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", +pData->pStrm, lenBuf); RUNLOG_VAR("%p", pData->pStrm); if(pData->pStrm != NULL){ @@ -842,6 +841,7 @@ RUNLOG_VAR("%p", pData->pStrm); FINALIZE; // TODO: clean up later } +#if 0 if(pData->fd != poBuf->fd) { // TODO: more efficient use for dynafiles CHKiRet(doFlush(pData)); @@ -854,6 +854,7 @@ RUNLOG_VAR("%p", pData->pStrm); CHKiRet(doFlush(pData)); } } +#endif finalize_it: RETiRet; @@ -875,45 +876,42 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) * check if it still is ok or a new file needs to be created */ if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + CHKiRet(prepareDynFile(pData, ppString[1], iMsgOpts)); + } else { /* "regular", non-dynafile */ + if(pData->pStrm == NULL) { + CHKiRet(prepareFile(pData, pData->f_fname)); + } } /* create the message based on format specified */ CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); finalize_it: + if(iRet != RS_RET_OK) { + /* in v5, we shall return different states for message-cause failur (but only there!) */ + iRet = RS_RET_SUSPENDED; + } RETiRet; } BEGINcreateInstance CODESTARTcreateInstance - pData->fd = -1; - CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); - pData->poBuf->lenBuf = OUTBUF_LEN; + pData->pStrm = NULL; + //CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); + //pData->poBuf->lenBuf = OUTBUF_LEN; finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ - if(pData->pStrm != NULL) { - strm.Destruct(&pData->pStrm); - } + //DEL! doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); - } else if(pData->fd != -1) - close(pData->fd); - free(pData->poBuf); + } else if(pData->pStrm != NULL) + strm.Destruct(&pData->pStrm); + //free(pData->poBuf); ENDfreeInstance @@ -1019,7 +1017,7 @@ CODESTARTparseSelectorAct */ prepareFile(pData, pData->f_fname); - if(pData->fd < 0 ) { + if(pData->pStrm == NULL) { DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); } @@ -1063,9 +1061,9 @@ CODESTARTdoHUP dynaFileFreeCacheEntries(pData); pData->iCurrElt = -1; /* invalidate current element */ } else { - if(pData->fd != -1) { - close(pData->fd); - pData->fd = -1; + if(pData->pStrm != NULL) { + strm.Destruct(&pData->pStrm); + pData->pStrm = NULL; } } ENDdoHUP -- cgit From 84b786753744d9f7a002665571a428283feae248 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Jun 2009 18:41:28 +0200 Subject: cleanup in preparation of next steps now cleand up omfile and straighted out some things. The only commented-out code left is code that must be moved/merged to the stream class, my next target. --- runtime/stream.h | 24 +++++++- tools/omfile.c | 174 ++++--------------------------------------------------- 2 files changed, 33 insertions(+), 165 deletions(-) diff --git a/runtime/stream.h b/runtime/stream.h index e4871611..7eb386fb 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -19,7 +19,29 @@ * can easily be persistet. The bottom line is that it makes much sense to * use this class whereever possible as its features may grow in the future. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * An important note on writing gzip format via zlib (kept anonymous + * by request): + * + * -------------------------------------------------------------------------- + * We'd like to make sure the output file is in full gzip format + * (compatible with gzip -d/zcat etc). There is a flag in how the output + * is initialized within zlib to properly add the gzip wrappers to the + * output. (gzip is effectively a small metadata wrapper around raw + * zstream output.) + * + * I had written an old bit of code to do this - the documentation on + * deflatInit2() was pretty tricky to nail down on this specific feature: + * + * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, + * int memLevel, int strategy); + * + * I believe "31" would be the value for the "windowBits" field that you'd + * want to try: + * + * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + * -------------------------------------------------------------------------- + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * diff --git a/tools/omfile.c b/tools/omfile.c index dd585893..f196d602 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -20,29 +20,6 @@ * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * - * An important note on writing gzip format via zlib (kept anonymous - * by request): - * - * -------------------------------------------------------------------------- - * We'd like to make sure the output file is in full gzip format - * (compatible with gzip -d/zcat etc). There is a flag in how the output - * is initialized within zlib to properly add the gzip wrappers to the - * output. (gzip is effectively a small metadata wrapper around raw - * zstream output.) - * - * I had written an old bit of code to do this - the documentation on - * deflatInit2() was pretty tricky to nail down on this specific feature: - * - * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, - * int memLevel, int strategy); - * - * I believe "31" would be the value for the "windowBits" field that you'd - * want to try: - * - * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - * -------------------------------------------------------------------------- - * - * * This file is part of rsyslog. * * Rsyslog is free software: you can redistribute it and/or modify @@ -105,22 +82,10 @@ DEFobjCurrIf(strm) struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ strm_t *pStrm; /* our output stream */ - time_t lastUsed; /* for LRU - last access */ + time_t lastUsed; /* for LRU - last access */ // TODO: perforamcne change to counter (see other comment!) }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; -/* the output buffer structure */ -// TODO: later make this part of the dynafile cache! -#define OUTBUF_LEN 128 // TODO: make dynamic! -typedef struct { - uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ - size_t lenBuf; /* max size of buffer */ - size_t iBuf; /* current buffer index */ - /* elements for zip writing */ - z_stream zStrm; - char zipBuf[OUTBUF_LEN]; -} outbuf_t; - #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ @@ -144,7 +109,6 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - //outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -152,10 +116,10 @@ typedef struct _instanceData { eTypePIPE } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ - int fCreateMode; /* file creation mode for open() */ + int fCreateMode; /* file creation mode for open() */ // TODO: shuffle down to stream class int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ - int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ + int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -452,14 +416,8 @@ prepareFile(instanceData *pData, uchar *newFileName) { int fd; DEFiRet; -#if 0 - if(pData->fileType == eTypePIPE) { - fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); - FINALIZE; /* we are done in this case */ - } -#endif - // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 + // TODO: handle TTY/PIPE case! (in stream.c!) 2009-06-04 if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ @@ -512,8 +470,6 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); -dbgprintf("static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); -RUNLOG_VAR("%d", pData->iIOBufSize); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); @@ -574,6 +530,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) /* ok, no luck. Now let's search the table if we find a matching spot. * While doing so, we also prepare for creation of a new one. */ + pData->iCurrElt = -1; /* invalid current element pointer */ iFirstFree = -1; /* not yet found */ iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */ ttOldest = time(NULL) + 1; /* there must always be an older one */ @@ -603,7 +560,6 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) iFirstFree = pData->iCurrCacheSize++; } - pData->iCurrElt = -1; if(iFirstFree == -1) { dynaFileDelCacheEntry(pCache, iOldest, 0); iFirstFree = iOldest; /* this one *is* now free ;) */ @@ -642,6 +598,7 @@ finalize_it: #if 0 +// TODO: mirgrate code below to stream class (update its write handler, which is not great!) /* physically write the file */ static rsRetVal @@ -738,85 +695,6 @@ again: finalize_it: RETiRet; } - - -/* write the output buffer in zip mode - * This means we compress it first and then do a physical write. - */ -static rsRetVal -doZipWrite(instanceData *pData) -{ - outbuf_t *poBuf; - z_stream zstrm; - int zRet; /* zlib return state */ - DEFiRet; - assert(pData != NULL); - - poBuf = pData->poBuf; /* use as a shortcut */ - zstrm = poBuf->zStrm; /* another shortcut */ - - /* allocate deflate state */ - zstrm.zalloc = Z_NULL; - zstrm.zfree = Z_NULL; - zstrm.opaque = Z_NULL; - /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&zstrm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } -RUNLOG_STR("deflateInit2() done successfully\n"); - - /* now doing the compression */ - zstrm.avail_in = poBuf->iBuf; - zstrm.next_in = (Bytef*) poBuf->pszBuf; - /* run deflate() on input until output buffer not full, finish - compression if all of source has been read in */ - do { - dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); - zstrm.avail_out = OUTBUF_LEN; - zstrm.next_out = (Bytef*) poBuf->zipBuf; - zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ - dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); - assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - zstrm.avail_out)); - } while (zstrm.avail_out == 0); - assert(zstrm.avail_in == 0); /* all input will be used */ - -RUNLOG_STR("deflate() should be done successfully\n"); - - zRet = zlibw.DeflateEnd(&zstrm); - if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } -RUNLOG_STR("deflateEnd() done successfully\n"); - -finalize_it: - RETiRet; -} - - -/* flush the output buffer - */ -static rsRetVal -doFlush(instanceData *pData) -{ - DEFiRet; - assert(pData != NULL); - - if(pData->poBuf->iBuf == 0) - FINALIZE; /* nothing to write, but make this a valid case */ - - if(0) { // zlib enabled! - CHKiRet(doZipWrite(pData)); - } else { - CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); - } - -finalize_it: - RETiRet; -} #endif @@ -832,30 +710,12 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pData != NULL); ASSERT(pszBuf != NULL); -dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", -pData->pStrm, lenBuf); - -RUNLOG_VAR("%p", pData->pStrm); +dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); FINALIZE; // TODO: clean up later } -#if 0 - if(pData->fd != poBuf->fd) { - // TODO: more efficient use for dynafiles - CHKiRet(doFlush(pData)); - poBuf->fd = pData->fd; - } - - for(i = 0 ; i < lenBuf ; ++i) { - poBuf->pszBuf[poBuf->iBuf++] = pszBuf[i]; - if(poBuf->iBuf == poBuf->lenBuf) { - CHKiRet(doFlush(pData)); - } - } -#endif - finalize_it: RETiRet; } @@ -883,7 +743,6 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) } } - /* create the message based on format specified */ CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); finalize_it: @@ -898,20 +757,15 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->pStrm = NULL; - //CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); - //pData->poBuf->lenBuf = OUTBUF_LEN; -finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - //DEL! doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->pStrm != NULL) strm.Destruct(&pData->pStrm); - //free(pData->poBuf); ENDfreeInstance @@ -969,14 +823,9 @@ CODESTARTparseSelectorAct pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ - /* we now allocate the cache table. We use calloc() intentionally, as we - * need all pointers to be initialized to NULL pointers. - */ - if((pData->dynCache = (dynaFileCacheEntry**) - calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n"); - } + /* we now allocate the cache table */ + CHKmalloc(pData->dynCache = (dynaFileCacheEntry**) + calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))); break; case '|': @@ -1022,9 +871,6 @@ CODESTARTparseSelectorAct errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); } } - -dbgprintf("in init static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); -RUNLOG_VAR("%d", pData->iIOBufSize); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct -- cgit From 8d1e2e496c6a4a4d40d1e8604c746e0d32013536 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Jun 2009 18:39:06 +0200 Subject: improved error handling and added fsync() support ... restoring missing functionality after the restructuring of imfile. As a side-effect, this also lays the foundation for even more reliable queue engine operations (but this is not yet done). --- runtime/stream.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- runtime/stream.h | 5 ++- tools/omfile.c | 4 +- 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/runtime/stream.c b/runtime/stream.c index a4844d2b..8e2f87dc 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -150,6 +150,12 @@ static rsRetVal strmCloseFile(strm_t *pThis) close(pThis->fd); // TODO: error check pThis->fd = -1; + if(pThis->fdDir != -1) { + /* close associated directory handle, if it is open */ + close(pThis->fdDir); + pThis->fdDir = -1; + } + if(pThis->bDeleteOnClose) { unlink((char*) pThis->pszCurrFName); // TODO: check returncode } @@ -395,10 +401,11 @@ finalize_it: BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! */ pThis->iCurrFNum = 1; pThis->fd = -1; + pThis->fdDir = -1; pThis->iUngetC = -1; pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); - pThis->tOpenMode = 0600; /* TODO: make configurable */ + pThis->tOpenMode = 0600; ENDobjConstruct(strm) @@ -428,6 +435,19 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } } + /* if we are aset to sync, we must obtain a file handle to the directory for fsync() purposes */ + if(pThis->bSync) { + pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY); + if(pThis->fdDir == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + // TODO: log an error message? think so... + DBGPRINTF("error %d opening directory file for fsync() use - fsync for directory disabled: %s\n", + errno, errStr); + } + } + finalize_it: RETiRet; } @@ -478,6 +498,81 @@ finalize_it: } + +/* issue write() api calls until either the buffer is completely + * written or an error occured (it may happen that multiple writes + * are required, what is perfectly legal. On exit, *pLenBuf contains + * the number of bytes actually written. + * rgerhards, 2009-06-08 + */ +static rsRetVal +doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) +{ + ssize_t lenBuf; + ssize_t iTotalWritten; + ssize_t iWritten; + char *pWriteBuf; + DEFiRet; + + lenBuf = *pLenBuf; + pWriteBuf = (char*) pBuf; + iTotalWritten = 0; + do { + iWritten = write(fd, pWriteBuf, lenBuf); + if(iWritten < 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("log file (%d) write error %d: %s\n", fd, err, errStr); + if(err == EINTR) { + /*NO ERROR, just continue */; + } else { + ABORT_FINALIZE(RS_RET_ERR); + // TODO: cover more error cases! + } + } + /* advance buffer to next write position */ + iTotalWritten += iWritten; + lenBuf -= iWritten; + pWriteBuf += iWritten; + } while(lenBuf > 0); /* Warning: do..while()! */ + +finalize_it: + *pLenBuf -= iTotalWritten; + RETiRet; +} + + +/* sync the file to disk, so that any unwritten data is persisted. This + * also syncs the directory and thus makes sure that the file survives + * fatal failure. -- rgerhards, 2009-06-08 + */ +static rsRetVal +syncFile(strm_t *pThis) +{ + int ret; + DEFiRet; + + DBGPRINTF("syncing file %d\n", pThis->fd); + ret = fdatasync(pThis->fd); + if(ret != 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("sync failed for file %d with error (%d): %s - ignoring\n", + pThis->fd, err, errStr); + } + // TODO: check error! + + if(pThis->fdDir != -1) { + ret = fsync(pThis->fdDir); +dbgprintf("sync on dir (fd %d) requested, return code %d\n", pThis->fdDir, ret); + } + + RETiRet; +} + + /* physically write to the output file. the provided data is ready for * writing (e.g. zipped if we are requested to do that). * rgerhards, 2009-06-04 @@ -485,17 +580,17 @@ finalize_it: static rsRetVal strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { + size_t iWritten; DEFiRet; - int iWritten; ASSERT(pThis != NULL); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); - iWritten = write(pThis->fd, pBuf, lenBuf); - dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, iWritten); - /* TODO: handle error case -- rgerhards, 2008-01-07 */ + iWritten = lenBuf; + CHKiRet(doWriteCall(pThis->fd, pBuf, &iWritten)); + dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, (int) iWritten); /* Now indicate buffer empty again. We do this in any case, because there * is no way we could react more intelligently to an error during write. @@ -511,6 +606,10 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) if(pThis->pUsrWCntr != NULL) *pThis->pUsrWCntr += iWritten; + if(pThis->bSync) { + CHKiRet(syncFile(pThis)); + } + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) CHKiRet(strmCheckNextOutputFile(pThis)); @@ -759,6 +858,7 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) @@ -1040,6 +1140,7 @@ CODESTARTobjQueryInterface(strm) pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index 7eb386fb..a66108b7 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -103,10 +103,12 @@ typedef struct strm_s { int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ - size_t sIOBufSize;/* size of IO buffer */ + int bSync; /* sync this file after every write? */ + size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; int fd; /* the file descriptor, -1 if closed */ + int fdDir; /* the directory's descriptor, in case bSync is requested (-1 if closed) */ uchar *pszCurrFName; /* name of current file (if open) */ uchar *pIOBuf; /* io Buffer */ size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ @@ -148,6 +150,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, tOpenMode, mode_t); INTERFACEpropSetMeth(strm, sType, strmType_t); INTERFACEpropSetMeth(strm, iZipLevel, int); + INTERFACEpropSetMeth(strm, bSync, int); INTERFACEpropSetMeth(strm, sIOBufSize, size_t); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tools/omfile.c b/tools/omfile.c index f196d602..e3f7af19 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -116,7 +116,7 @@ typedef struct _instanceData { eTypePIPE } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ - int fCreateMode; /* file creation mode for open() */ // TODO: shuffle down to stream class + int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! @@ -472,6 +472,8 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); + CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); + CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); -- cgit From 9704f129f72ec9ece11aeccea4bbf0cbccb116cb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Jun 2009 19:00:18 +0200 Subject: added capability to fsync() queue disk files for enhanced reliability also adds speed, because you do no longer need to run the whole file system in sync mode. New testbench and new config directives: - $MainMsgQueueSyncQueueFiles - $ActionQueueSyncQueueFiles --- ChangeLog | 5 +++++ action.c | 4 ++++ doc/queues.html | 6 +++++- runtime/queue.c | 5 +++++ runtime/queue.h | 2 ++ tests/Makefile.am | 3 +++ tests/diskqueue-fsync.sh | 15 +++++++++++++++ tests/testsuites/diskqueue-fsync.conf | 17 +++++++++++++++++ tools/syslogd.c | 4 ++++ 9 files changed, 60 insertions(+), 1 deletion(-) create mode 100755 tests/diskqueue-fsync.sh create mode 100644 tests/testsuites/diskqueue-fsync.conf diff --git a/ChangeLog b/ChangeLog index 8fc4194d..57bb6e7e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,9 +12,14 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - reduced max value for $DynaFileCacheSize to 1,000 (the former maximum of 10,000 really made no sense, even 1,000 is very high, but we like to keep the user in control ;)). +- added capability to fsync() queue disk files for enhanced reliability + (also add's speed, because you do no longer need to run the whole file + system in sync mode) - added configuration commands (see doc for explanations) * $OMFileZipLevel * $OMFileIOBufferSize + * $MainMsgQueueSyncQueueFiles + * $ActionQueueSyncQueueFiles --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/action.c b/action.c index 51620fce..8bdb6dec 100644 --- a/action.c +++ b/action.c @@ -72,6 +72,7 @@ static int iActionQueueNumWorkers = 1; /* number of worker threads for the mm static uchar *pszActionQFName = NULL; /* prefix for the main message queue file */ static int64 iActionQueMaxFileSize = 1024*1024; static int iActionQPersistUpdCnt = 0; /* persist queue info every n updates */ +static int bActionQSyncQeueFiles = 0; /* sync queue files */ static int iActionQtoQShutdown = 0; /* queue shutdown */ static int iActionQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iActionQtoEnq = 2000; /* timeout for queue enque */ @@ -151,6 +152,7 @@ actionResetQueueParams(void) iActionQueueNumWorkers = 1; /* number of worker threads for the mm queue above */ iActionQueMaxFileSize = 1024*1024; iActionQPersistUpdCnt = 0; /* persist queue info every n updates */ + bActionQSyncQeueFiles = 0; iActionQtoQShutdown = 0; /* queue shutdown */ iActionQtoActShutdown = 1000; /* action shutdown (in phase 2) */ iActionQtoEnq = 2000; /* timeout for queue enque */ @@ -273,6 +275,7 @@ actionConstructFinalize(action_t *pThis) setQPROP(qqueueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); setQPROPstr(qqueueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); setQPROP(qqueueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); + setQPROP(qqueueSetbSyncQueueFiles, "$ActionQueueSyncQueueFiles", bActionQSyncQeueFiles); setQPROP(qqueueSettoQShutdown, "$ActionQueueTimeoutShutdown", iActionQtoQShutdown ); setQPROP(qqueueSettoActShutdown, "$ActionQueueTimeoutActionCompletion", iActionQtoActShutdown); setQPROP(qqueueSettoWrkShutdown, "$ActionQueueWorkerTimeoutThreadShutdown", iActionQtoWrkShutdown); @@ -838,6 +841,7 @@ actionAddCfSysLineHdrl(void) CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardmark", 0, eCmdHdlrInt, NULL, &iActionQDiscardMark, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardseverity", 0, eCmdHdlrInt, NULL, &iActionQDiscardSeverity, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iActionQPersistUpdCnt, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bActionQSyncQeueFiles, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetype", 0, eCmdHdlrGetWord, setActionQueType, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iActionQueueNumWorkers, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoQShutdown, NULL)); diff --git a/doc/queues.html b/doc/queues.html index 4a9509a0..45ce1bd1 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -115,7 +115,11 @@ isolation. This is currently selected by specifying different $WorkDirectory< config directives before the queue creation statement.

    To create a disk queue, use the "$<object>QueueType Disk" config directive. Checkpoint intervals can be specified via "$<object>QueueCheckpointInterval", -with 0 meaning no checkpoints.

    +with 0 meaning no checkpoints. Note that disk-based queues can be made very reliable +by issuing a (f)sync after each write operation. Starting with version 4.3.2, this can +be requested via "<object>QueueSyncQueueFiles on/off with the +default being off. Activating this option has a performance penalty, so it should +not be turned on without reason.

    In-Memory Queues

    In-memory queue mode is what most people have on their mind when they think about computing queues. Here, the enqueued data elements are held in memory. diff --git a/runtime/queue.c b/runtime/queue.c index 3532a145..aa8e6c21 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -294,6 +294,7 @@ qqueueStartDA(qqueue_t *pThis) CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); + CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); @@ -817,6 +818,7 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) ; } else { CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pWrite, pThis->bSyncQueueFiles)); CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); @@ -824,6 +826,7 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); CHKiRet(strm.Construct(&pThis->tVars.disk.pRead)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pRead, pThis->bSyncQueueFiles)); CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); CHKiRet(strm.SetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); @@ -1924,6 +1927,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) CHKiRet(strm.Construct(&psQIF)); CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC)); + CHKiRet(strm.SetbSync(psQIF, pThis->bSyncQueueFiles)); CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); CHKiRet(strm.ConstructFinalize(psQIF)); @@ -2279,6 +2283,7 @@ finalize_it: /* some simple object access methods */ +DEFpropSetMeth(qqueue, bSyncQueueFiles, int) DEFpropSetMeth(qqueue, iPersistUpdCnt, int) DEFpropSetMeth(qqueue, iDeqtWinFromHr, int) DEFpropSetMeth(qqueue, iDeqtWinToHr, int) diff --git a/runtime/queue.h b/runtime/queue.h index a267862d..07f134aa 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -73,6 +73,7 @@ typedef struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ + int bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ @@ -186,6 +187,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); +PROTOTYPEpropSetMeth(qqueue, bSyncQueueFiles, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinToHr, int); PROTOTYPEpropSetMeth(qqueue, toQShutdown, long); diff --git a/tests/Makefile.am b/tests/Makefile.am index a95139f2..3d555fc3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -5,6 +5,7 @@ TESTS = $(TESTRUNS) cfg.sh \ validation-run.sh \ imtcp-multiport.sh \ diskqueue.sh \ + diskqueue-fsync.sh \ manytcp.sh \ queue-persist.sh @@ -59,6 +60,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ fieldtest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ + diskqueue-fsync.sh \ + testsuites/diskqueue-fsync.conf \ imtcp-multiport.sh \ testsuites/imtcp-multiport.conf \ manytcp.sh \ diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh new file mode 100755 index 00000000..f558b491 --- /dev/null +++ b/tests/diskqueue-fsync.sh @@ -0,0 +1,15 @@ +# Test for disk-only queue mode (with fsync for queue files) +# This test checks if queue files can be correctly written +# and read back, but it does not test the transition from +# memory to disk mode for DA queues. +# added 2009-06-09 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo testing queue disk-only mode, fsync case +source $srcdir/diag.sh init +source $srcdir/diag.sh startup diskqueue-fsync.conf +# 5000 messages should be enough - the disk fsync test is very slow! +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 5000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/diskqueue-fsync.conf b/tests/testsuites/diskqueue-fsync.conf new file mode 100644 index 00000000..0a02c6ce --- /dev/null +++ b/tests/testsuites/diskqueue-fsync.conf @@ -0,0 +1,17 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueSyncQueueFiles on +$MainMsgQueueTimeoutShutdown 10000 +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tools/syslogd.c b/tools/syslogd.c index 206b79e8..19ece349 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -297,6 +297,7 @@ static queueType_t MainMsgQueType = QUEUETYPE_FIXED_ARRAY; /* type of the main m static uchar *pszMainMsgQFName = NULL; /* prefix for the main message queue file */ static int64 iMainMsgQueMaxFileSize = 1024*1024; static int iMainMsgQPersistUpdCnt = 0; /* persist queue info every n updates */ +static int bMainMsgQSyncQeueFiles = 0; /* sync queue files on every write? */ static int iMainMsgQtoQShutdown = 0; /* queue shutdown */ static int iMainMsgQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iMainMsgQtoEnq = 2000; /* timeout for queue enque */ @@ -359,6 +360,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iMainMsgQueMaxFileSize = 1024 * 1024; iMainMsgQueueNumWorkers = 1; iMainMsgQPersistUpdCnt = 0; + bMainMsgQSyncQeueFiles = 0; iMainMsgQtoQShutdown = 0; iMainMsgQtoActShutdown = 1000; iMainMsgQtoEnq = 2000; @@ -2715,6 +2717,7 @@ init(void) setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); + setQPROP(qqueueSetbSyncQueueFiles, "$MainMsgQueueSyncQueueFiles", bMainMsgQSyncQeueFiles); setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); @@ -3084,6 +3087,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardmark", 0, eCmdHdlrInt, NULL, &iMainMsgQDiscardMark, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardseverity", 0, eCmdHdlrSeverity, NULL, &iMainMsgQDiscardSeverity, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iMainMsgQPersistUpdCnt, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bMainMsgQSyncQeueFiles, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetype", 0, eCmdHdlrGetWord, setMainMsgQueType, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iMainMsgQueueNumWorkers, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iMainMsgQtoQShutdown, NULL)); -- cgit From 6f4e3c4e4c85acdcf58969970484a54639ecc8f9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 16:49:14 +0200 Subject: restructered code in perparation for multiple rule set support ... this was long overdue, and I finlly tackeld it. It turned out to be more complex than I initially thought. The next step now probably is to actually implement multiple rule sets and the beauty that comes with them. --- dirty.h | 2 + runtime/Makefile.am | 4 + runtime/conf.c | 102 +++++---- runtime/conf.h | 2 +- runtime/linkedlist.c | 2 +- runtime/obj-types.h | 5 +- runtime/obj.h | 2 +- runtime/rsyslog.c | 8 + runtime/rsyslog.h | 19 +- runtime/rule.c | 431 +++++++++++++++++++++++++++++++++++++ runtime/rule.h | 74 +++++++ runtime/ruleset.c | 369 ++++++++++++++++++++++++++++++++ runtime/ruleset.h | 56 +++++ runtime/stream.c | 3 - tests/diskqueue-fsync.sh | 2 + tests/ourtail.c | 2 + tests/runtime-dummy.c | 4 + tests/validation-run.sh | 7 +- tools/syslogd.c | 537 +++++------------------------------------------ tools/syslogd.h | 56 +---- 20 files changed, 1093 insertions(+), 594 deletions(-) create mode 100644 runtime/rule.c create mode 100644 runtime/rule.h create mode 100644 runtime/ruleset.c create mode 100644 runtime/ruleset.h diff --git a/dirty.h b/dirty.h index 6d585753..bab15485 100644 --- a/dirty.h +++ b/dirty.h @@ -33,6 +33,7 @@ rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len int parseRFCSyslogMsg(msg_t *pMsg, int flags); int parseLegacySyslogMsg(msg_t *pMsg, int flags); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ +char* getFIOPName(unsigned iFIOP); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); @@ -56,6 +57,7 @@ extern int bReduceRepeatMsgs; extern int bDropTrailingLF; extern uchar cCCEscapeChar; extern int bEscapeCCOnRcv; +extern ruleset_t *pCurrRuleset; #ifdef USE_NETZIP /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 81422d64..de0daac4 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -67,6 +67,10 @@ librsyslog_la_SOURCES = \ vmop.h \ queue.c \ queue.h \ + ruleset.c \ + ruleset.h \ + rule.c \ + rule.h \ cfsysline.c \ cfsysline.h \ \ diff --git a/runtime/conf.c b/runtime/conf.c index 7cdcf5ec..e0ed3d6a 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -69,13 +69,15 @@ #include "expr.h" #include "ctok.h" #include "ctok_token.h" +#include "rule.h" +#include "ruleset.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN #endif /* forward definitions */ -static rsRetVal cfline(uchar *line, selector_t **pfCurr); +static rsRetVal cfline(uchar *line, rule_t **pfCurr); static rsRetVal processConfFile(uchar *pConfFile); @@ -87,6 +89,8 @@ DEFobjCurrIf(ctok_token) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) +DEFobjCurrIf(rule) +DEFobjCurrIf(ruleset) static int iNbrActions; /* number of actions the running config has. Needs to be init on ReInitConf() */ @@ -395,7 +399,7 @@ processConfFile(uchar *pConfFile) DEFiRet; int iLnNbr = 0; FILE *cf; - selector_t *fCurr = NULL; + rule_t *pCurrRule = NULL; uchar *p; uchar cbuf[CFGLNSIZ]; uchar *cline; @@ -429,7 +433,6 @@ processConfFile(uchar *pConfFile) * TODO: review the code at whole - this is highly suspect (but will go away * once we do the rest of RainerScript). */ - /* was: strcpy((char*)cline, (char*)p); */ for( i = 0 ; p[i] != '\0' ; ++i) { cline[i] = p[i]; } @@ -453,7 +456,7 @@ processConfFile(uchar *pConfFile) /* we now have the complete line, and are positioned at the first non-whitespace * character. So let's process it */ - if(cfline(cbuf, &fCurr) != RS_RET_OK) { + if(cfline(cbuf, &pCurrRule) != RS_RET_OK) { /* we log a message, but otherwise ignore the error. After all, the next * line can be correct. -- rgerhards, 2007-08-02 */ @@ -467,16 +470,18 @@ processConfFile(uchar *pConfFile) } /* we probably have one selector left to be added - so let's do that now */ - CHKiRet(selectorAddList(fCurr)); + if(pCurrRule != NULL) { + CHKiRet(ruleset.AddRule(pCurrRuleset, &pCurrRule)); + } /* close the configuration file */ - (void) fclose(cf); + fclose(cf); finalize_it: if(iRet != RS_RET_OK) { char errStr[1024]; - if(fCurr != NULL) - selectorDestruct(fCurr); + if(pCurrRule != NULL) + rule.Destruct(&pCurrRule); rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("error %d processing config file '%s'; os error (if any): %s\n", @@ -588,7 +593,7 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int * rgerhards 2005-09-15 */ /* GPLv3 - stems back to sysklogd */ -static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) { uchar *p; register uchar *q; @@ -603,17 +608,17 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f ASSERT(pline != NULL); ASSERT(*pline != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(pRule, rule); dbgprintf(" - traditional PRI filter\n"); errno = 0; /* keep strerror_r() stuff out of logerror messages */ - f->f_filter_type = FILTER_PRI; + pRule->f_filter_type = FILTER_PRI; /* Note: file structure is pre-initialized to zero because it was * created with calloc()! */ for (i = 0; i <= LOG_NFACILITIES; i++) { - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } /* scan through the list of selectors */ @@ -668,32 +673,32 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f for (i = 0; i <= LOG_NFACILITIES; i++) { if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] |= (1<f_filterData.f_pmask[i >> 3] |= (1<f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] |= (1<f_filterData.f_pmask[i >> 3] |= (1<f_filter_type = FILTER_EXPR; /* if we come to over here, pline starts with "if ". We just skip that part. */ @@ -823,7 +827,7 @@ finalize_it: * of the action part. A pointer to that beginnig is passed back to the caller. * rgerhards 2005-09-15 */ -static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) { rsParsObj *pPars; cstr_t *pCSCompOp; @@ -1014,10 +1018,10 @@ static rsRetVal cflineProcessTagSelector(uchar **pline) /* read the filter part of a configuration line and store the filter - * in the supplied selector_t + * in the supplied rule_t * rgerhards, 2007-08-01 */ -static rsRetVal cflineDoFilter(uchar **pp, selector_t *f) +static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) { DEFiRet; @@ -1110,17 +1114,15 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) /* Process a configuration file line in traditional "filter selector" format - * or one that builds upon this format. + * or one that builds upon this format. Note that ppRule may be a NULL pointer, + * which is valid and happens if there is no previous line (right at the start + * of the master config file!). */ -static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) +static rsRetVal +cflineClassic(uchar *p, rule_t **ppRule) { DEFiRet; action_t *pAction; - selector_t *fCurr; - - ASSERT(pfCurr != NULL); - - fCurr = *pfCurr; /* lines starting with '&' have no new filters and just add * new actions to the currently processed selector. @@ -1138,16 +1140,18 @@ static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) * selector is NULL, which means we do not need to care about it at * all. -- rgerhards, 2007-08-01 */ - CHKiRet(selectorAddList(fCurr)); - CHKiRet(selectorConstruct(&fCurr)); /* create "fresh" selector */ - CHKiRet(cflineDoFilter(&p, fCurr)); /* pull filters */ + if(*ppRule != NULL) { + CHKiRet(ruleset.AddRule(pCurrRuleset, ppRule)); + } + CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ + CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ + CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } CHKiRet(cflineDoAction(&p, &pAction)); - CHKiRet(llAppend(&fCurr->llActList, NULL, (void*) pAction)); + CHKiRet(llAppend(&(*ppRule)->llActList, NULL, (void*) pAction)); finalize_it: - *pfCurr = fCurr; RETiRet; } @@ -1157,11 +1161,12 @@ finalize_it: * rgerhards, 2007-08-01 */ static rsRetVal -cfline(uchar *line, selector_t **pfCurr) +cfline(uchar *line, rule_t **pfCurr) { DEFiRet; ASSERT(line != NULL); +if(*pfCurr != NULL){ ISOBJ_TYPE_assert(*pfCurr, rule);} dbgprintf("cfline: '%s'\n", line); @@ -1180,6 +1185,7 @@ cfline(uchar *line, selector_t **pfCurr) break; default: iRet = cflineClassic(line, pfCurr); +ISOBJ_TYPE_assert(*pfCurr, rule); break; } @@ -1254,6 +1260,8 @@ CODESTARTObjClassExit(conf) objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); + objRelease(rule, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDObjClassExit(conf) @@ -1269,6 +1277,8 @@ BEGINAbstractObjClassInit(conf, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANG CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* TODO: make this dependcy go away! */ + CHKiRet(objUse(rule, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); ENDObjClassInit(conf) /* vi:set ai: diff --git a/runtime/conf.h b/runtime/conf.h index 2494d4dc..f29de1ba 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -35,7 +35,7 @@ BEGINinterface(conf) /* name must also be changed in ENDinterface macro! */ rsRetVal (*cfsysline)(uchar *p); rsRetVal (*doModLoad)(uchar **pp, __attribute__((unused)) void* pVal); rsRetVal (*doIncludeLine)(uchar **pp, __attribute__((unused)) void* pVal); - rsRetVal (*cfline)(uchar *line, selector_t **pfCurr); + rsRetVal (*cfline)(uchar *line, rule_t **pfCurr); rsRetVal (*processConfFile)(uchar *pConfFile); rsRetVal (*ReInitConf)(void); rsRetVal (*GetNbrActActions)(int *); diff --git a/runtime/linkedlist.c b/runtime/linkedlist.c index 8f842e43..cc095f6e 100644 --- a/runtime/linkedlist.c +++ b/runtime/linkedlist.c @@ -398,7 +398,7 @@ rsRetVal llExecFunc(linkedList_t *pThis, rsRetVal (*pFunc)(void*, void*), void* */ llCookie = llCookiePrev; } else if (iRet != RS_RET_OK) { - goto finalize_it; + FINALIZE; } llCookiePrev = llCookie; } diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 914c2f2c..23f74590 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -105,12 +105,13 @@ struct obj_s { /* the dummy struct that each derived class can be casted to */ # define ISOBJ_TYPE_assert(pObj, objType) \ do { \ ASSERT(pObj != NULL); \ - ASSERT((unsigned) ((obj_t*) (pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ if(strcmp((char*)(((obj_t*)pObj)->pObjInfo->pszID), #objType)) { \ dbgprintf("%s:%d ISOBJ assert failure: invalid object type, expected '%s' " \ - "actual '%s'\n", __FILE__, __LINE__, #objType, (((obj_t*)pObj)->pObjInfo->pszID)); \ + "actual '%s', cookie: %X\n", __FILE__, __LINE__, #objType, \ + (((obj_t*)pObj)->pObjInfo->pszID), ((obj_t*)(pObj))->iObjCooCKiE); \ assert(0); /* trigger assertion, messge we already have */ \ } \ + ASSERT((unsigned) ((obj_t*)(pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ } while(0) #else /* non-debug mode, no checks but much faster */ # define BEGINobjInstance obj_t objData diff --git a/runtime/obj.h b/runtime/obj.h index dc04203b..3973a16e 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -68,7 +68,7 @@ #define objSerializePTR(strm, propName, propType) \ CHKiRet(obj.SerializeProp(strm, (uchar*) #propName, PROPTYPE_##propType, (void*) pThis->propName)); #define DEFobjStaticHelpers \ - static objInfo_t *pObjInfoOBJ = NULL; \ + static objInfo_t __attribute__((unused)) *pObjInfoOBJ = NULL; \ DEFobjCurrIf(obj) diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 81550f12..3496bb0d 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -77,6 +77,8 @@ #include "conf.h" #include "glbl.h" #include "errmsg.h" +#include "rule.h" +#include "ruleset.h" /* forward definitions */ static rsRetVal dfltErrLogger(int, uchar *errMsg); @@ -172,6 +174,10 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "expr"; CHKiRet(exprClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "rule"; + CHKiRet(ruleClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ruleset"; + CHKiRet(rulesetClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); @@ -204,6 +210,8 @@ rsrtExit(void) /* do actual de-init only if we are the last runtime user */ confClassExit(); glblClassExit(); + rulesetClassExit(); + ruleClassExit(); objClassExit(); /* *THIS* *MUST/SHOULD?* always be the first class initilizer being called (except debug)! */ } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index f93592b7..b587554e 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -62,7 +62,9 @@ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ typedef struct thrdInfo thrdInfo_t; typedef struct obj_s obj_t; -typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ +typedef struct ruleset_s ruleset_t; +typedef struct rule_s rule_t; +//typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ typedef struct NetAddr netAddr_t; typedef struct netstrms_s netstrms_t; typedef struct netstrm_s netstrm_t; @@ -112,6 +114,16 @@ typedef enum { eFLOWCTL_FULL_DELAY = 2 /**< delay possible for extended period of time */ } flowControl_t; +/* filter operations */ +typedef enum { + FIOP_NOP = 0, /* do not use - No Operation */ + FIOP_CONTAINS = 1, /* contains string? */ + FIOP_ISEQUAL = 2, /* is (exactly) equal? */ + FIOP_STARTSWITH = 3, /* starts with a string? */ + FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ + FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ +} fiop_t; + /* The error codes below are orginally "borrowed" from * liblogging. As such, we reserve values up to -2999 @@ -367,6 +379,11 @@ typedef enum rsObjectID rsObjID; # define O_CLOEXEC 0 #endif +/* some constants */ +// TODO: do we really need them - if not, delete -- rgerhards, 2009-06-10 +#define IGNORE_ERROR_CODES 1 +#define ABORT_ON_ERROR 0 + /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); diff --git a/runtime/rule.c b/runtime/rule.c new file mode 100644 index 00000000..fa56dc3b --- /dev/null +++ b/runtime/rule.c @@ -0,0 +1,431 @@ +/* rule.c - rsyslog's rule object + * + * See file comment in rule.c for the overall structure of rule processing. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "action.h" +#include "rule.h" +#include "errmsg.h" +#include "vm.h" +#include "var.h" +#include "srUtils.h" +#include "dirty.h" /* for getFIOPName */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(expr) +DEFobjCurrIf(var) +DEFobjCurrIf(vm) + +/* iterate over all actions, this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + return llExecFunc(&pThis->llActList, pFunc, pParam); +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +typedef struct processMsgDoActions_s { + int bPrevWasSuspended; /* was the previous action suspended? */ + msg_t *pMsg; +} processMsgDoActions_t; +DEFFUNC_llExecFunc(processMsgDoActions) +{ + DEFiRet; + rsRetVal iRetMod; /* return value of module - we do not always pass that back */ + action_t *pAction = (action_t*) pData; + processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; + + assert(pAction != NULL); + + if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { + dbgprintf("not calling action because the previous one is not suspended\n"); + ABORT_FINALIZE(RS_RET_OK); + } + + iRetMod = actionCallAction(pAction, pDoActData->pMsg); + if(iRetMod == RS_RET_DISCARDMSG) { + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } else if(iRetMod == RS_RET_SUSPENDED) { + /* indicate suspension for next module to be called */ + pDoActData->bPrevWasSuspended = 1; + } else { + pDoActData->bPrevWasSuspended = 0; + } + +finalize_it: + RETiRet; +} + + +/* This functions looks at the given message and checks if it matches the + * provided filter condition. + */ +static rsRetVal +shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) +{ + DEFiRet; + unsigned short pbMustBeFreed; + char *pszPropVal; + int bRet = 0; + vm_t *pVM = NULL; + var_t *pResult = NULL; + + ISOBJ_TYPE_assert(pRule, rule); + assert(pMsg != NULL); + + /* we first have a look at the global, BSD-style block filters (for tag + * and host). Only if they match, we evaluate the actual filter. + * rgerhards, 2005-10-18 + */ + if(pRule->eHostnameCmpMode == HN_NO_COMP) { + /* EMPTY BY INTENSION - we check this value first, because + * it is the one most often used, so this saves us time! + */ + } else if(pRule->eHostnameCmpMode == HN_COMP_MATCH) { + if(rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '+%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } else { /* must be -hostname */ + if(!rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '-%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } + + if(pRule->pCSProgNameComp != NULL) { + int bInv = 0, bEqv = 0, offset = 0; + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp)) == '-') { + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp) + 1) == '-') + offset = 1; + else { + bInv = 1; + offset = 1; + } + } + if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) + bEqv = 1; + + if((!bEqv && !bInv) || (bEqv && bInv)) { + /* not equal or inverted selection, so we are already done... */ + dbgprintf("programname filter '%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg)); + FINALIZE; + } + } + + /* done with the BSD-style block filters */ + + if(pRule->f_filter_type == FILTER_PRI) { + /* skip messages that are incorrect priority */ + if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ + ((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) + bRet = 0; + else + bRet = 1; + } else if(pRule->f_filter_type == FILTER_EXPR) { + CHKiRet(vm.Construct(&pVM)); + CHKiRet(vm.ConstructFinalize(pVM)); + CHKiRet(vm.SetMsg(pVM, pMsg)); + CHKiRet(vm.ExecProg(pVM, pRule->f_filterData.f_expr->pVmprg)); + CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); + dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + /* VM is destructed on function exit */ + bRet = (pResult->val.num) ? 1 : 0; + } else { + assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ + pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.pCSPropName, &pbMustBeFreed); + + /* Now do the compares (short list currently ;)) */ + switch(pRule->f_filterData.prop.operation ) { + case FIOP_CONTAINS: + if(rsCStrLocateInSzStr(pRule->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) + bRet = 1; + break; + case FIOP_ISEQUAL: + if(rsCStrSzStrCmp(pRule->f_filterData.prop.pCSCompValue, + (uchar*) pszPropVal, strlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_STARTSWITH: + if(rsCStrSzStrStartsWithCStr(pRule->f_filterData.prop.pCSCompValue, + (uchar*) pszPropVal, strlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_REGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 0, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 1, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + default: + /* here, it handles NOP (for performance reasons) */ + assert(pRule->f_filterData.prop.operation == FIOP_NOP); + bRet = 1; /* as good as any other default ;) */ + break; + } + + /* now check if the value must be negated */ + if(pRule->f_filterData.prop.isNegated) + bRet = (bRet == 1) ? 0 : 1; + + if(Debug) { + dbgprintf("Filter: check for property '%s' (value '%s') ", + rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSPropName), + pszPropVal); + if(pRule->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("%s '%s': %s\n", + getFIOPName(pRule->f_filterData.prop.operation), + rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue), + bRet ? "TRUE" : "FALSE"); + } + + /* cleanup */ + if(pbMustBeFreed) + free(pszPropVal); + } + +finalize_it: + /* destruct in any case, not just on error, but it makes error handling much easier */ + if(pVM != NULL) + vm.Destruct(&pVM); + + if(pResult != NULL) + var.Destruct(&pResult); + + *bProcessMsg = bRet; + RETiRet; +} + + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(rule_t *pThis, msg_t *pMsg) +{ + int bProcessMsg; + processMsgDoActions_t DoActData; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, rule); + assert(pMsg != NULL); + + /* first check the filters... */ + CHKiRet(shouldProcessThisMessage(pThis, pMsg, &bProcessMsg)); + if(bProcessMsg) { + DoActData.pMsg = pMsg; + DoActData.bPrevWasSuspended = 0; + CHKiRet(llExecFunc(&pThis->llActList, processMsgDoActions, (void*)&DoActData)); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(rule) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(rule) + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +ruleConstructFinalize(rule_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + + /* note: actionDestruct is from action.c API! */ + CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +/* destructor for the rule object */ +BEGINobjDestruct(rule) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(rule) + if(pThis->pCSHostnameComp != NULL) + rsCStrDestruct(&pThis->pCSHostnameComp); + if(pThis->pCSProgNameComp != NULL) + rsCStrDestruct(&pThis->pCSProgNameComp); + + if(pThis->f_filter_type == FILTER_PROP) { + if(pThis->f_filterData.prop.pCSPropName != NULL) + rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); + if(pThis->f_filterData.prop.pCSCompValue != NULL) + rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); + if(pThis->f_filterData.prop.regex_cache != NULL) + rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); + } else if(pThis->f_filter_type == FILTER_EXPR) { + if(pThis->f_filterData.f_expr != NULL) + expr.Destruct(&pThis->f_filterData.f_expr); + } + + llDestroy(&pThis->llActList); + free(pThis); + + return RS_RET_OK; +ENDobjDestruct(rule) + + +/* helper to DebugPrint, to print out all actions via + * the llExecFunc() facility. + */ +DEFFUNC_llExecFunc(dbgPrintInitInfoAction) +{ + DEFiRet; + iRet = actionDbgPrint((action_t*) pData); + dbgprintf("\n"); + RETiRet; +} + + +/* debugprint for the rule object */ +BEGINobjDebugPrint(rule) /* be sure to specify the object type also in END and CODESTART macros! */ + int i; +CODESTARTobjDebugPrint(rule) + dbgoprint((obj_t*) pThis, "rsyslog rule:\n"); + if(pThis->pCSProgNameComp != NULL) + dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(pThis->pCSProgNameComp)); + if(pThis->eHostnameCmpMode != HN_NO_COMP) + dbgprintf("hostname: %s '%s'\n", + pThis->eHostnameCmpMode == HN_COMP_MATCH ? + "only" : "allbut", + rsCStrGetSzStrNoNULL(pThis->pCSHostnameComp)); + if(pThis->f_filter_type == FILTER_PRI) { + for (i = 0; i <= LOG_NFACILITIES; i++) + if (pThis->f_filterData.f_pmask[i] == TABLE_NOPRI) + dbgprintf(" X "); + else + dbgprintf("%2X ", pThis->f_filterData.f_pmask[i]); + } else if(pThis->f_filter_type == FILTER_EXPR) { + dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); + } else { + dbgprintf("PROPERTY-BASED Filter:\n"); + dbgprintf("\tProperty.: '%s'\n", + rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSPropName)); + dbgprintf("\tOperation: "); + if(pThis->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("'%s'\n", getFIOPName(pThis->f_filterData.prop.operation)); + dbgprintf("\tValue....: '%s'\n", + rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue)); + dbgprintf("\tAction...: "); + } + + dbgprintf("\nActions:\n"); + llExecFunc(&pThis->llActList, dbgPrintInitInfoAction, NULL); /* actions */ + + dbgprintf("\n"); +ENDobjDebugPrint(rule) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(rule) +CODESTARTobjQueryInterface(rule) + if(pIf->ifVersion != ruleCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = ruleConstruct; + pIf->ConstructFinalize = ruleConstructFinalize; + pIf->Destruct = ruleDestruct; + pIf->DebugPrint = ruleDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->ProcessMsg = processMsg; +finalize_it: +ENDobjQueryInterface(rule) + + +/* Exit the rule class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(rule, OBJ_IS_CORE_MODULE) /* class, version */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(expr, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vm, CORE_COMPONENT); +ENDObjClassExit(rule) + + +/* Initialize the rule class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(rule, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(expr, CORE_COMPONENT)); + CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, ruleDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, ruleConstructFinalize); +ENDObjClassInit(rule) + +/* vi:set ai: + */ diff --git a/runtime/rule.h b/runtime/rule.h new file mode 100644 index 00000000..96be9e30 --- /dev/null +++ b/runtime/rule.h @@ -0,0 +1,74 @@ +/* The rule object. + * + * This implements rules within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULE_H +#define INCLUDED_RULE_H + +#include "linkedlist.h" +#include "regexp.h" +#include "expr.h" + +/* the rule object */ +struct rule_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + /* filter properties */ + enum { + FILTER_PRI = 0, /* traditional PRI based filer */ + FILTER_PROP = 1, /* extended filter, property based */ + FILTER_EXPR = 2 /* extended filter, expression based */ + } f_filter_type; + EHostnameCmpMode eHostnameCmpMode; + cstr_t *pCSHostnameComp; /* hostname to check */ + cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ + union { + u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ + struct { + cstr_t *pCSPropName; + fiop_t operation; + regex_t *regex_cache; /* cache for compiled REs, if such are used */ + cstr_t *pCSCompValue; /* value to "compare" against */ + char isNegated; /* actually a boolean ;) */ + } prop; + expr_t *f_expr; /* expression object */ + } f_filterData; + + linkedList_t llActList; /* list of configured actions */ +}; + +/* interfaces */ +BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(rule); + rsRetVal (*Construct)(rule_t **ppThis); + rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(rule_t **ppThis); + rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); + rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg); +ENDinterface(rule) +#define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(rule); + +#endif /* #ifndef INCLUDED_RULE_H */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c new file mode 100644 index 00000000..d135074f --- /dev/null +++ b/runtime/ruleset.c @@ -0,0 +1,369 @@ +/* ruleset.c - rsyslog's ruleset object + * + * We have a two-way structure of linked lists: one global linked list + * (llAllRulesets) hold alls rule sets that we know. Included in each + * list is a list of rules (which contain a list of actions, but that's + * a different story). + * + * Usually, only a single rule set is executed. However, there exist some + * situations where all rules must be iterated over, for example on HUP. Thus, + * we also provide interfaces to do that. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "ruleset.h" +#include "rule.h" +#include "errmsg.h" +#include "unicode-helper.h" + +static rsRetVal debugPrintAll(void); // TODO: remove! + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(rule) + +linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ + +/* ---------- linked-list key handling functions ---------- */ + +/* destructor for linked list keys. + */ +static rsRetVal keyDestruct(void __attribute__((unused)) *pData) +{ + free(pData); + return RS_RET_OK; +} + + +/* ---------- END linked-list key handling functions ---------- */ + + +/* dirver to iterate over all of this ruleset actions */ +typedef struct iterateAllActions_s { + rsRetVal (*pFunc)(void*, void*); + void *pParam; +} iterateAllActions_t; +DEFFUNC_llExecFunc(doIterateRulesetActions) +{ + DEFiRet; + rule_t* pRule = (rule_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over all actions of THIS rule set. + */ +static rsRetVal +iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&llRulesets, doIterateRulesetActions, ¶ms)); + +finalize_it: + RETiRet; +} + + +/* dirver to iterate over all actions */ +DEFFUNC_llExecFunc(doIterateAllActions) +{ + DEFiRet; + ruleset_t* pThis = (ruleset_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over ALL actions present in the WHOLE system. + * this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + +finalize_it: + RETiRet; +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +DEFFUNC_llExecFunc(processMsgDoRules) +{ + ISOBJ_TYPE_assert(pData, rule); + return rule.ProcessMsg((rule_t*) pData, (msg_t*) pParam); +} + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(ruleset_t *pThis, msg_t *pMsg) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + assert(pMsg != NULL); + + CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); + +finalize_it: + if(iRet == RS_RET_DISCARDMSG) + iRet = RS_RET_OK; + + RETiRet; +} + +/* Add a new rule to the end of the current rule set. We do a number + * of checks and ignore the rule if it does not pass them. + */ +static rsRetVal +addRule(ruleset_t *pThis, rule_t **ppRule) +{ + int iActionCnt; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, ruleset); + ISOBJ_TYPE_assert(*ppRule, rule); + + CHKiRet(llGetNumElts(&(*ppRule)->llActList, &iActionCnt)); + if(iActionCnt == 0) { + errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); + rule.Destruct(ppRule); + } else { + CHKiRet(llAppend(&pThis->llRules, NULL, *ppRule)); + dbgprintf("selector line successfully processed\n"); + } + +finalize_it: + RETiRet; +} + + +/* set name for ruleset */ +static rsRetVal setName(ruleset_t *pThis, uchar *pszName) +{ + DEFiRet; + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); + +finalize_it: + RETiRet; +} + + +/* destructor we need to destruct rules inside our linked list contents. + */ +static rsRetVal +doRuleDestruct(void *pData) +{ + rule_t *pRule = (rule_t *) pData; + DEFiRet; + rule.Destruct(&pRule); + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(ruleset) /* be sure to specify the object type also in END macro! */ + CHKiRet(llInit(&pThis->llRules, doRuleDestruct, NULL, NULL)); +finalize_it: +ENDobjConstruct(ruleset) + + +/* ConstructionFinalizer + * This also adds the rule set to the list of all known rulesets. + */ +static rsRetVal +rulesetConstructFinalize(ruleset_t *pThis) +{ + uchar *keyName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + + /* we must duplicate our name, as the key destructer would also + * free it, resulting in a double-free. It's also cleaner to have + * two separate copies. + */ + CHKmalloc(keyName = ustrdup(pThis->pszName)); + CHKiRet(llAppend(&llRulesets, keyName, pThis)); + +finalize_it: + RETiRet; +} + + +/* destructor for the ruleset object */ +BEGINobjDestruct(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(ruleset) + dbgprintf("destructing ruleset %p, name %p\n", pThis, pThis->pszName); + llDestroy(&pThis->llRules); + free(pThis->pszName); +ENDobjDestruct(ruleset) + +/* this is a special destructor for the linkedList class. LinkedList does NOT + * provide a pointer to the pointer, but rather the raw pointer itself. So we + * must map this, otherwise the destructor will abort. + */ +static rsRetVal +rulesetDestructForLinkedList(void *pData) +{ + ruleset_t *pThis = (ruleset_t*) pData; + return rulesetDestruct(&pThis); +} + + +/* destruct ALL rule sets that reside in the system. This must + * be callable before unloading this module as the module may + * not be unloaded before unload of the actions is required. This is + * kind of a left-over from previous logic and may be optimized one + * everything runs stable again. -- rgerhards, 2009-06-10 + */ +static rsRetVal +destructAllActions(void) +{ + DEFiRet; + + CHKiRet(llDestroy(&llRulesets)); + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + +finalize_it: + RETiRet; +} + +/* helper for debugPrint(), initiates rule printing */ +DEFFUNC_llExecFunc(doDebugPrintRule) +{ + return rule.DebugPrint((rule_t*) pData); +} +/* debugprint for the ruleset object */ +BEGINobjDebugPrint(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(ruleset) + dbgoprint((obj_t*) pThis, "rsyslog ruleset %s:\n", pThis->pszName); + llExecFunc(&pThis->llRules, doDebugPrintRule, NULL); +ENDobjDebugPrint(ruleset) + + +/* helper for debugPrintAll(), prints a single ruleset */ +DEFFUNC_llExecFunc(doDebugPrintAll) +{ + return rulesetDebugPrint((ruleset_t*) pData); +} +/* debug print all rulesets + */ +static rsRetVal +debugPrintAll(void) +{ + DEFiRet; + dbgprintf("All Rulesets:\n"); + llExecFunc(&llRulesets, doDebugPrintAll, NULL); + dbgprintf("End of Rulesets.\n"); + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(ruleset) +CODESTARTobjQueryInterface(ruleset) + if(pIf->ifVersion != rulesetCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = rulesetConstruct; + pIf->ConstructFinalize = rulesetConstructFinalize; + pIf->Destruct = rulesetDestruct; + pIf->DebugPrint = rulesetDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->DestructAllActions = destructAllActions; + pIf->AddRule = addRule; + pIf->ProcessMsg = processMsg; + pIf->SetName = setName; + pIf->DebugPrintAll = debugPrintAll; +finalize_it: +ENDobjQueryInterface(ruleset) + + +/* Exit the ruleset class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ + llDestroy(&llRulesets); + objRelease(errmsg, CORE_COMPONENT); + objRelease(rule, CORE_COMPONENT); +ENDObjClassExit(ruleset) + + +/* Initialize the ruleset class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(rule, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, rulesetDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, rulesetConstructFinalize); + + /* prepare global data */ + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); +ENDObjClassInit(ruleset) + +/* vi:set ai: + */ diff --git a/runtime/ruleset.h b/runtime/ruleset.h new file mode 100644 index 00000000..b609e6b3 --- /dev/null +++ b/runtime/ruleset.h @@ -0,0 +1,56 @@ +/* The ruleset object. + * + * This implements rulesets within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULESET_H +#define INCLUDED_RULESET_H + +#include "linkedlist.h" + +/* the ruleset object */ +struct ruleset_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ + uchar *pszName; /* name of our ruleset */ +}; + +/* interfaces */ +BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(ruleset); + rsRetVal (*DebugPrintAll)(void); + rsRetVal (*Construct)(ruleset_t **ppThis); + rsRetVal (*ConstructFinalize)(ruleset_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(ruleset_t **ppThis); + rsRetVal (*IterateAllActions)(rsRetVal (*pFunc)(void*, void*), void* pParam); + rsRetVal (*DestructAllActions)(void); + rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); + rsRetVal (*ProcessMsg)(ruleset_t *pThis, msg_t *pMsg); + rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); +ENDinterface(ruleset) +#define rulesetCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(ruleset); + +#endif /* #ifndef INCLUDED_RULESET_H */ diff --git a/runtime/stream.c b/runtime/stream.c index 8e2f87dc..49d29e0e 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -650,7 +650,6 @@ doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } -RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ zstrm.avail_in = lenBuf; @@ -668,14 +667,12 @@ RUNLOG_STR("deflateInit2() done successfully\n"); } while (zstrm.avail_out == 0); assert(zstrm.avail_in == 0); /* all input will be used */ -RUNLOG_STR("deflate() should be done successfully\n"); zRet = zlibw.DeflateEnd(&zstrm); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } -RUNLOG_STR("deflateEnd() done successfully\n"); finalize_it: RETiRet; diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh index f558b491..8d2cb709 100755 --- a/tests/diskqueue-fsync.sh +++ b/tests/diskqueue-fsync.sh @@ -5,6 +5,8 @@ # added 2009-06-09 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: +echo diskqueue-fsync test seems to have some issues +exit 1 echo testing queue disk-only mode, fsync case source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue-fsync.conf diff --git a/tests/ourtail.c b/tests/ourtail.c index 6781b5fe..4e8a6412 100644 --- a/tests/ourtail.c +++ b/tests/ourtail.c @@ -40,4 +40,6 @@ int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) for( ; c != EOF ; c = getchar()) putchar(c); + + return 0; } diff --git a/tests/runtime-dummy.c b/tests/runtime-dummy.c index 9cddd913..38e6bba1 100644 --- a/tests/runtime-dummy.c +++ b/tests/runtime-dummy.c @@ -25,7 +25,9 @@ * * A copy of the GPL can be found in the file "COPYING" in this distribution. */ +#include "config.h" #include +#include "rsyslog.h" int bReduceRepeatMsgs = 0; int repeatinterval = 30; @@ -37,5 +39,7 @@ void cflineClassic(void) {}; void selectorAddList(void) {}; void selectorConstruct(void) {}; void selectorDestruct(void) {}; +void getFIOPName(void) {}; +ruleset_t *pCurrRuleset; /* these are required by some dynamically loaded modules */ diff --git a/tests/validation-run.sh b/tests/validation-run.sh index 10981290..2e922283 100755 --- a/tests/validation-run.sh +++ b/tests/validation-run.sh @@ -22,17 +22,18 @@ # A copy of the GPL can be found in the file "COPYING" in this distribution. #set -x echo "testing a failed configuration verification run" -../tools/rsyslogd -u2 -c3 -N1 -f$srcdir/testsuites/invalid.conf +../tools/rsyslogd -dn -u2 -c4 -N1 -f$srcdir/testsuites/invalid.conf -M../runtime/.libs:../.libs if [ $? -ne 1 ]; then exit 1 fi echo testing a valid config verification run -../tools/rsyslogd -u2 -c3 -N1 -f$srcdir/testsuites/valid.conf +../tools/rsyslogd -u2 -c4 -N1 -f$srcdir/testsuites/valid.conf -M../runtime/.libs:../.libs if [ $? -ne 0 ]; then exit 1 fi echo testing empty config file -../tools/rsyslogd -u2 -c3 -N1 -f/dev/null +../tools/rsyslogd -u2 -c4 -N1 -f/dev/null -M../runtime/.libs:../.libs if [ $? -ne 1 ]; then exit 1 fi +echo SUCCESS: validation run tests diff --git a/tools/syslogd.c b/tools/syslogd.c index 19ece349..183c7760 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -132,11 +132,13 @@ #include "queue.h" #include "stream.h" #include "conf.h" -#include "vm.h" #include "errmsg.h" #include "datetime.h" #include "parser.h" #include "unicode-helper.h" +#include "ruleset.h" +#include "rule.h" +#include "vm.h" /* definitions for objects we access */ DEFobjCurrIf(obj) @@ -144,10 +146,10 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) DEFobjCurrIf(conf) DEFobjCurrIf(expr) -DEFobjCurrIf(vm) -DEFobjCurrIf(var) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +DEFobjCurrIf(rule) +DEFobjCurrIf(ruleset) DEFobjCurrIf(net) /* TODO: make go away! */ @@ -246,7 +248,7 @@ int repeatinterval[2] = { 30, 60 }; /* # of secs before flush */ #define LIST_DELIMITER ':' /* delimiter between two hosts */ -struct filed *Files = NULL; /* read-only after init() (but beware of sigusr1!) */ +ruleset_t *pCurrRuleset; /* ruleset that is currently being processed */ static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */ @@ -313,7 +315,8 @@ static int iMainMsgQueueDeqtWinToHr = 25; /* hour begin of time frame when que /* support for simple textual representation of FIOP names * rgerhards, 2005-09-27 */ -static char* getFIOPName(unsigned iFIOP) +char* +getFIOPName(unsigned iFIOP) { char *pRet; switch(iFIOP) { @@ -396,7 +399,6 @@ static char **crunch_list(char *list); static void reapchild(); static void debug_switch(); static void sighup_handler(); -static void freeSelectors(void); static void processImInternal(void); @@ -431,67 +433,6 @@ diagGetMainMsgQSize(int *piSize) /* ------------------------------ end support functions for imdiag ------------------------------ */ -/* function to destruct a selector_t object - * rgerhards, 2007-08-01 - */ -rsRetVal -selectorDestruct(void *pVal) -{ - selector_t *pThis = (selector_t *) pVal; - - assert(pThis != NULL); - - if(pThis->pCSHostnameComp != NULL) - rsCStrDestruct(&pThis->pCSHostnameComp); - if(pThis->pCSProgNameComp != NULL) - rsCStrDestruct(&pThis->pCSProgNameComp); - - if(pThis->f_filter_type == FILTER_PROP) { - if(pThis->f_filterData.prop.pCSPropName != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); - if(pThis->f_filterData.prop.pCSCompValue != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); - if(pThis->f_filterData.prop.regex_cache != NULL) - rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); - } else if(pThis->f_filter_type == FILTER_EXPR) { - if(pThis->f_filterData.f_expr != NULL) - expr.Destruct(&pThis->f_filterData.f_expr); - } - - llDestroy(&pThis->llActList); - free(pThis); - - return RS_RET_OK; -} - - -/* function to construct a selector_t object - * rgerhards, 2007-08-01 - */ -rsRetVal -selectorConstruct(selector_t **ppThis) -{ - DEFiRet; - selector_t *pThis; - - assert(ppThis != NULL); - - if((pThis = (selector_t*) calloc(1, sizeof(selector_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); - -finalize_it: - if(iRet != RS_RET_OK) { - if(pThis != NULL) { - selectorDestruct(pThis); - } - } - *ppThis = pThis; - RETiRet; -} - - /* rgerhards, 2005-10-24: crunch_list is called only during option processing. So * it is never called once rsyslogd is running (not even when HUPed). This code * contains some exits, but they are considered safe because they only happen @@ -993,233 +934,6 @@ finalize_it: RETiRet; } -/* This functions looks at the given message and checks if it matches the - * provided filter condition. If so, it returns true, else it returns - * false. This is a helper to logmsg() and meant to drive the decision - * process if a message is to be processed or not. As I expect this - * decision code to grow more complex over time AND logmsg() is already - * a very lengthy function, I thought a separate function is more appropriate. - * 2005-09-19 rgerhards - * 2008-02-25 rgerhards: changed interface, now utilizes iRet, bProcessMsg - * returns is message should be procesed. - */ -static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProcessMsg) -{ - DEFiRet; - unsigned short pbMustBeFreed; - char *pszPropVal; - int bRet = 0; - vm_t *pVM = NULL; - var_t *pResult = NULL; - - assert(f != NULL); - assert(pMsg != NULL); - - /* we first have a look at the global, BSD-style block filters (for tag - * and host). Only if they match, we evaluate the actual filter. - * rgerhards, 2005-10-18 - */ - if(f->eHostnameCmpMode == HN_NO_COMP) { - /* EMPTY BY INTENSION - we check this value first, because - * it is the one most often used, so this saves us time! - */ - } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { - if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - dbgprintf("hostname filter '+%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } else { /* must be -hostname */ - if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - dbgprintf("hostname filter '-%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } - - if(f->pCSProgNameComp != NULL) { - int bInv = 0, bEqv = 0, offset = 0; - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp)) == '-') { - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp) + 1) == '-') - offset = 1; - else { - bInv = 1; - offset = 1; - } - } - if(!rsCStrOffsetSzStrCmp(f->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) - bEqv = 1; - - if((!bEqv && !bInv) || (bEqv && bInv)) { - /* not equal or inverted selection, so we are already done... */ - dbgprintf("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); - FINALIZE; - } - } - - /* done with the BSD-style block filters */ - - if(f->f_filter_type == FILTER_PRI) { - /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ - ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - } else if(f->f_filter_type == FILTER_EXPR) { - CHKiRet(vm.Construct(&pVM)); - CHKiRet(vm.ConstructFinalize(pVM)); - CHKiRet(vm.SetMsg(pVM, pMsg)); - CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); - CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); - /* VM is destructed on function exit */ - bRet = (pResult->val.num) ? 1 : 0; - } else { - assert(f->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, f->f_filterData.prop.pCSPropName, &pbMustBeFreed); - - /* Now do the compares (short list currently ;)) */ - switch(f->f_filterData.prop.operation ) { - case FIOP_CONTAINS: - if(rsCStrLocateInSzStr(f->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) - bRet = 1; - break; - case FIOP_ISEQUAL: - if(rsCStrSzStrCmp(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_STARTSWITH: - if(rsCStrSzStrStartsWithCStr(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - case FIOP_EREREGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - default: - /* here, it handles NOP (for performance reasons) */ - assert(f->f_filterData.prop.operation == FIOP_NOP); - bRet = 1; /* as good as any other default ;) */ - break; - } - - /* now check if the value must be negated */ - if(f->f_filterData.prop.isNegated) - bRet = (bRet == 1) ? 0 : 1; - - if(Debug) { - dbgprintf("Filter: check for property '%s' (value '%s') ", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName), - pszPropVal); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("%s '%s': %s\n", - getFIOPName(f->f_filterData.prop.operation), - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue), - bRet ? "TRUE" : "FALSE"); - } - - /* cleanup */ - if(pbMustBeFreed) - free(pszPropVal); - } - -finalize_it: - /* destruct in any case, not just on error, but it makes error handling much easier */ - if(pVM != NULL) - vm.Destruct(&pVM); - - if(pResult != NULL) - var.Destruct(&pResult); - - *bProcessMsg = bRet; - RETiRet; -} - - -/* helper to processMsg(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -typedef struct processMsgDoActions_s { - int bPrevWasSuspended; /* was the previous action suspended? */ - msg_t *pMsg; -} processMsgDoActions_t; -DEFFUNC_llExecFunc(processMsgDoActions) -{ - DEFiRet; - rsRetVal iRetMod; /* return value of module - we do not always pass that back */ - action_t *pAction = (action_t*) pData; - processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; - - assert(pAction != NULL); - - if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - dbgprintf("not calling action because the previous one is not suspended\n"); - ABORT_FINALIZE(RS_RET_OK); - } - - iRetMod = actionCallAction(pAction, pDoActData->pMsg); - if(iRetMod == RS_RET_DISCARDMSG) { - ABORT_FINALIZE(RS_RET_DISCARDMSG); - } else if(iRetMod == RS_RET_SUSPENDED) { - /* indicate suspension for next module to be called */ - pDoActData->bPrevWasSuspended = 1; - } else { - pDoActData->bPrevWasSuspended = 0; - } - -finalize_it: - RETiRet; -} - - -/* Process (consume) a received message. Calls the actions configured. - * rgerhards, 2005-10-13 - */ -static void -processMsg(msg_t *pMsg) -{ - selector_t *f; - int bContinue; - int bProcessMsg; - processMsgDoActions_t DoActData; - rsRetVal iRet; - - BEGINfunc - assert(pMsg != NULL); - - /* log the message to the particular outputs */ - - bContinue = 1; - for (f = Files; f != NULL && bContinue ; f = f->f_next) { - /* first check the filters... */ - iRet = shouldProcessThisMessage(f, pMsg, &bProcessMsg); - if(!bProcessMsg) { - continue; - } - - /* ok -- from here, we have action-specific code, nothing really selector-specific -- rger 2007-08-01 */ - DoActData.pMsg = pMsg; - DoActData.bPrevWasSuspended = 0; - if(llExecFunc(&f->llActList, processMsgDoActions, (void*)&DoActData) == RS_RET_DISCARDMSG) - bContinue = 0; - } - ENDfunc -} - /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE @@ -1237,7 +951,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); } - processMsg(pMsg); + ruleset.ProcessMsg(pCurrRuleset, pMsg); msgDestruct(&pMsg); RETiRet; @@ -1744,7 +1458,6 @@ reapchild() DEFFUNC_llExecFunc(flushRptdMsgsActions) { action_t *pAction = (action_t*) pData; - assert(pAction != NULL); BEGINfunc @@ -1767,20 +1480,12 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) } -/* This method flushes reapeat messages. +/* This method flushes repeat messages. */ static void doFlushRptdMsgs(void) { - register selector_t *f; - - /* see if we need to flush any "message repeated n times"... - * Note that this interferes with objects running on other threads. - * We are using appropriate locking inside the function to handle that. - */ - for (f = Files; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, flushRptdMsgsActions, NULL); - } + ruleset.IterateAllActions(flushRptdMsgsActions, NULL); } @@ -1971,6 +1676,16 @@ freeAllDynMemForTermination(void) } +/* Finalize and destruct all actions. + */ +static inline void +destructAllActions(void) +{ + ruleset.DestructAllActions(); + bHaveMainQueue = 0; // flag that internal messages need to be temporarily stored +} + + /* die() is called when the program shall end. This typically only occurs * during sigterm or during the initialization. * As die() is intended to shutdown rsyslogd, it is @@ -2021,7 +1736,7 @@ die(int sig) * repeated msgs. */ dbgprintf("Terminating outputs...\n"); - freeSelectors(); + destructAllActions(); dbgprintf("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); /* rger 2005-02-22 @@ -2057,7 +1772,7 @@ die(int sig) * rgerhards, 2007-08-03 * I have added some code now, but all that mod init/de-init should be moved to * init, so that modules are unloaded and reloaded on HUP to. Eventually it should go - * into freeSelectors() - but that needs to be seen. -- rgerhards, 2007-08-09 + * into destructAllActions() - but that needs to be seen. -- rgerhards, 2007-08-09 */ module.UnloadAndDestructAll(eMOD_LINK_ALL); @@ -2185,56 +1900,6 @@ static void doDropPrivUid(int iUid) } -/* helper to freeSelectors(), used with llExecFunc() to flush - * pending output. -- rgerhards, 2007-08-02 - * We do not need to lock the action object here as the processing - * queue is already empty and no other threads are running when - * we call this function. -- rgerhards, 2007-12-12 - */ -DEFFUNC_llExecFunc(freeSelectorsActions) -{ - action_t *pAction = (action_t*) pData; - - assert(pAction != NULL); - - /* flush any pending output */ - if(pAction->f_prevcount) { - actionWriteToAction(pAction); - } - - return RS_RET_OK; /* never fails ;) */ -} - - -/* Close all open log files and free selector descriptor array. - */ -static void freeSelectors(void) -{ - selector_t *f; - selector_t *fPrev; - - if(Files != NULL) { - dbgprintf("Freeing log structures.\n"); - - for(f = Files ; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, freeSelectorsActions, NULL); - } - - /* actions flushed and ready for destruction - so do that... */ - f = Files; - while (f != NULL) { - fPrev = f; - f = f->f_next; - selectorDestruct(fPrev); - } - - /* Reflect the deletion of the selectors linked list. */ - Files = NULL; - bHaveMainQueue = 0; - } -} - - /* helper to generateConfigDAG, to print out all actions via * the llExecFunc() facility. * rgerhards, 2007-08-02 @@ -2311,14 +1976,14 @@ DEFFUNC_llExecFunc(generateConfigDAGAction) static rsRetVal generateConfigDAG(uchar *pszDAGFile) { - selector_t *f; + //rule_t *f; FILE *fp; int iActUnit = 1; - int bHasFilter = 0; /* filter associated with this action unit? */ - int bHadFilter; - int i; + //int bHasFilter = 0; /* filter associated with this action unit? */ + //int bHadFilter; + //int i; struct dag_info dagInfo; - char *pszFilterName; + //char *pszFilterName; char szConnectingNode[64]; DEFiRet; @@ -2347,6 +2012,8 @@ generateConfigDAG(uchar *pszDAGFile) strcpy(szConnectingNode, "act0_0"); dagInfo.bDiscarded = 0; +/* TODO: re-enable! */ +#if 0 for(f = Files; f != NULL ; f = f->f_next) { /* BSD-Style filters are currently ignored */ bHadFilter = bHasFilter; @@ -2402,6 +2069,7 @@ generateConfigDAG(uchar *pszDAGFile) ++iActUnit; } +#endif fprintf(fp, "\t%s -> act%d_0\n", szConnectingNode, iActUnit); fprintf(fp, "\tact%d_0\t\t[label=discard shape=box]\n" @@ -2413,20 +2081,6 @@ finalize_it: } -/* helper to dbPrintInitInfo, to print out all actions via - * the llExecFunc() facility. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(dbgPrintInitInfoAction) -{ - DEFiRet; - iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); - - RETiRet; -} - - /* print debug information as part of init(). This pretty much * outputs the whole config of rsyslogd. I've moved this code * out of init() to clean it somewhat up. @@ -2434,46 +2088,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) */ static void dbgPrintInitInfo(void) { - selector_t *f; - int iSelNbr = 1; - int i; - - dbgprintf("\nActive selectors:\n"); - for (f = Files; f != NULL ; f = f->f_next) { - dbgprintf("Selector %d:\n", iSelNbr++); - if(f->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); - if(f->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", - f->eHostnameCmpMode == HN_COMP_MATCH ? - "only" : "allbut", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp)); - if(f->f_filter_type == FILTER_PRI) { - for (i = 0; i <= LOG_NFACILITIES; i++) - if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", f->f_filterData.f_pmask[i]); - } else if(f->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); - } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName)); - dbgprintf("\tOperation: "); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue)); - dbgprintf("\tAction...: "); - } - - dbgprintf("\nActions:\n"); - llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - - dbgprintf("\n"); - } + ruleset.DebugPrintAll(); dbgprintf("\n"); if(bDebugPrintTemplateList) tplPrintList(); @@ -2553,13 +2168,13 @@ startInputModules(void) static rsRetVal init(void) { - DEFiRet; rsRetVal localRet; int iNbrActions; int bHadConfigErr = 0; char cbuf[BUFSIZ]; char bufStartUpMsg[512]; struct sigaction sigAct; + DEFiRet; thrdTerminateAll(); /* stop all running input threads - TODO: reconsider location! */ @@ -2580,7 +2195,7 @@ init(void) /* Close all open log files and free log descriptor array. This also frees * all output-modules instance data. */ - freeSelectors(); + destructAllActions(); /* Unload all non-static modules */ dbgprintf("Unloading non-static modules.\n"); @@ -2601,6 +2216,11 @@ init(void) conf.ReInitConf(); + // TODO: move to the right place + ruleset.Construct(&pCurrRuleset); + ruleset.SetName(pCurrRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); + ruleset.ConstructFinalize(pCurrRuleset); + /* open the configuration file */ localRet = conf.processConfFile(ConfFile); CHKiRet(conf.GetNbrActActions(&iNbrActions)); @@ -2622,23 +2242,24 @@ init(void) * We ignore any errors while doing this - we would be lost anyhow... */ errmsg.LogError(0, NO_ERRCODE, "EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file!"); - selector_t *f = NULL; /* note: we previously used _POSIY_TTY_NAME_MAX+1, but this turned out to be * too low on linux... :-S -- rgerhards, 2008-07-28 */ char szTTYNameBuf[128]; - conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &f); - conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &f); - conf.cfline((uchar*)"*.PANIC\t*", &f); - conf.cfline((uchar*)"syslog.*\troot", &f); + rule_t *pRule; + CHKiRet(rule.Construct(&pRule)); + conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &pRule); + conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &pRule); + conf.cfline((uchar*)"*.PANIC\t*", &pRule); + conf.cfline((uchar*)"syslog.*\troot", &pRule); if(ttyname_r(0, szTTYNameBuf, sizeof(szTTYNameBuf)) == 0) { snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); - conf.cfline((uchar*)cbuf, &f); + conf.cfline((uchar*)cbuf, &pRule); } else { dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } - selectorAddList(f); + ruleset.AddRule(pCurrRuleset, &pRule); } legacyOptsHook(); @@ -2777,53 +2398,6 @@ finalize_it: } -/* add a completely-processed selector (after config line parsing) to - * the linked list of selectors. We now need to check - * if it has any actions associated and, if so, link it to the linked - * list. If it has nothing associated with it, we can simply discard - * it. - * We have one special case during initialization: then, the current - * selector is NULL, which means we do not need to care about it at - * all. -- rgerhards, 2007-08-01 - */ -rsRetVal -selectorAddList(selector_t *f) -{ - DEFiRet; - int iActionCnt; - - static selector_t *nextp = NULL; /* TODO: make this go away (see comment below) */ - - if(f != NULL) { - CHKiRet(llGetNumElts(&f->llActList, &iActionCnt)); - if(iActionCnt == 0) { - errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); - selectorDestruct(f); - } else { - /* successfully created an entry */ - dbgprintf("selector line successfully processed\n"); - /* TODO: we should use the linked list class for the selector list, else we need to add globals - * ... well nextp could be added temporarily... - * Thanks to varmojfekoj for having the idea to just use "Files" to make this - * code work. I had actually forgotten to fix the code here before moving to 1.18.0. - * And, of course, I also did not migrate the selector_t structure to the linked list class. - * However, that should still be one of the very next things to happen. - * rgerhards, 2007-08-06 - */ - if(Files == NULL) { - Files = f; - } else { - nextp->f_next = f; - } - nextp = f; - } - } - -finalize_it: - RETiRet; -} - - /* set the main message queue mode * rgerhards, 2008-01-03 */ @@ -2911,7 +2485,6 @@ DEFFUNC_llExecFunc(doHUPActions) static inline void doHUP(void) { - selector_t *f; char buf[512]; snprintf(buf, sizeof(buf) / sizeof(char), @@ -2926,9 +2499,7 @@ doHUP(void) init(); /* main queue is stopped as part of init() */ } else { DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); - for(f = Files; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, doHUPActions, NULL); - } + ruleset.IterateAllActions(doHUPActions, NULL); } } @@ -3285,14 +2856,14 @@ InitGlobalClasses(void) CHKiRet(objUse(errmsg, CORE_COMPONENT)); pErrObj = "module"; CHKiRet(objUse(module, CORE_COMPONENT)); - pErrObj = "var"; - CHKiRet(objUse(var, CORE_COMPONENT)); pErrObj = "datetime"; CHKiRet(objUse(datetime, CORE_COMPONENT)); - pErrObj = "vm"; - CHKiRet(objUse(vm, CORE_COMPONENT)); pErrObj = "expr"; CHKiRet(objUse(expr, CORE_COMPONENT)); + pErrObj = "rule"; + CHKiRet(objUse(rule, CORE_COMPONENT)); + pErrObj = "ruleset"; + CHKiRet(objUse(ruleset, CORE_COMPONENT)); pErrObj = "conf"; CHKiRet(objUse(conf, CORE_COMPONENT)); @@ -3336,10 +2907,10 @@ GlobalClassExit(void) /* first, release everything we used ourself */ objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ objRelease(conf, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); + objRelease(rule, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ - objRelease(vm, CORE_COMPONENT); - objRelease(var, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); /* TODO: implement the rest of the deinit */ diff --git a/tools/syslogd.h b/tools/syslogd.h index 8b9bd131..9bcfb2d2 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -28,67 +28,17 @@ #include "action.h" #include "linkedlist.h" #include "expr.h" +#include "net.h" /* TODO: remove when you remove isAllowedSender from here! */ #ifndef _PATH_CONSOLE #define _PATH_CONSOLE "/dev/console" #endif - -/* This structure represents the files that will have log - * copies printed. - * RGerhards 2004-11-08: Each instance of the filed structure - * describes what I call an "output channel". This is important - * to mention as we now allow database connections to be - * present in the filed structure. If helps immensely, if we - * think of it as the abstraction of an output channel. - * rgerhards, 2005-10-26: The structure below provides ample - * opportunity for non-thread-safety. Each of the variable - * accesses must be carefully evaluated, many of them probably - * be guarded by mutexes. But beware of deadlocks... - * rgerhards, 2007-08-01: as you can see, the structure has shrunk pretty much. I will - * remove some of the comments some time. It's still the structure that controls much - * of the processing that goes on in syslogd, but it now has lots of helpers. - */ -struct filed { - struct filed *f_next; /* next in linked list */ - /* filter properties */ - enum { - FILTER_PRI = 0, /* traditional PRI based filer */ - FILTER_PROP = 1, /* extended filter, property based */ - FILTER_EXPR = 2 /* extended filter, expression based */ - } f_filter_type; - EHostnameCmpMode eHostnameCmpMode; - cstr_t *pCSHostnameComp; /* hostname to check */ - cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ - union { - u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ - struct { - cstr_t *pCSPropName; - enum { - FIOP_NOP = 0, /* do not use - No Operation */ - FIOP_CONTAINS = 1, /* contains string? */ - FIOP_ISEQUAL = 2, /* is (exactly) equal? */ - FIOP_STARTSWITH = 3, /* starts with a string? */ - FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ - FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ - } operation; - regex_t *regex_cache; /* cache for compiled REs, if such are used */ - cstr_t *pCSCompValue; /* value to "compare" against */ - char isNegated; /* actually a boolean ;) */ - } prop; - expr_t *f_expr; /* expression object */ - } f_filterData; - - linkedList_t llActList; /* list of configured actions */ -}; - - -#include "net.h" /* TODO: remove when you remoe isAllowedSender from here! */ void untty(void); -rsRetVal selectorConstruct(selector_t **ppThis); +rsRetVal selectorConstruct(ruleset_t **ppThis); rsRetVal selectorDestruct(void *pVal); -rsRetVal selectorAddList(selector_t *f); +rsRetVal selectorAddList(rule_t *f); /* the following prototypes should go away once we have an input * module interface -- rgerhards, 2007-12-12 */ -- cgit From 0290be816e683a711fc3c3a73a2ec79b804cfc8b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 16:55:00 +0200 Subject: fixed small nit in build system --- tests/Makefile.am | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 3d555fc3..dbaf85f0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -73,8 +73,6 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ - daqueue-persist.sh \ - daqueue-persist-drvr.sh \ testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ -- cgit From 1dbdee7c774dd20e4653efc6871ddef5adce2785 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 18:05:24 +0200 Subject: fixed a small memory leak... and another problem, both introduced today. Also did some general cleanup. --- runtime/obj.c | 39 ++++++++++++++++----------------------- runtime/rule.c | 3 --- runtime/wti.c | 3 +-- runtime/wtp.c | 3 +-- 4 files changed, 18 insertions(+), 30 deletions(-) diff --git a/runtime/obj.c b/runtime/obj.c index f5ec66b2..8b9c9c83 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -87,6 +87,7 @@ #include "modules.h" #include "errmsg.h" #include "cfsysline.h" +#include "unicode-helper.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ @@ -145,8 +146,8 @@ InfoConstruct(objInfo_t **ppThis, uchar *pszID, int iObjVers, ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pThis->pszID = pszID; - pThis->lenID = strlen((char*)pszID); - pThis->pszName = (uchar*)strdup((char*)pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ + pThis->lenID = ustrlen(pszID); + pThis->pszName = ustrdup(pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ pThis->iObjVers = iObjVers; pThis->QueryIF = pQueryIF; pThis->pModInfo = pModInfo; @@ -177,8 +178,7 @@ InfoDestruct(objInfo_t **ppThis) pThis = *ppThis; assert(pThis != NULL); - if(pThis->pszName != NULL) - free(pThis->pszName); + free(pThis->pszName); free(pThis); *ppThis = NULL; @@ -206,9 +206,7 @@ DestructObjSelf(obj_t *pThis) DEFiRet; ISOBJ_assert(pThis); - if(pThis->pszName != NULL) { - free(pThis->pszName); - } + free(pThis->pszName); RETiRet; } @@ -321,31 +319,31 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr switch(propType) { case PROPTYPE_PSZ: pszBuf = (uchar*) pUsr; - lenBuf = strlen((char*) pszBuf); + lenBuf = ustrlen(pszBuf); vType = VARTYPE_STR; break; case PROPTYPE_SHORT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((short*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((int*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_LONG: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((long*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT64: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((int64*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_CSTR: @@ -380,7 +378,7 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr /* cookie */ CHKiRet(strm.WriteChar(pStrm, COOKIE_PROPLINE)); /* name */ - CHKiRet(strm.Write(pStrm, pszPropName, strlen((char*)pszPropName))); + CHKiRet(strm.Write(pStrm, pszPropName, ustrlen(pszPropName))); CHKiRet(strm.WriteChar(pStrm, ':')); /* type */ CHKiRet(strm.WriteLong(pStrm, (int) vType)); @@ -804,7 +802,7 @@ Deserialize(void *ppObj, uchar *pszTypeExpected, strm_t *pStrm, rsRetVal (*fFixu } } while(iRetLocal != RS_RET_OK); - if(rsCStrSzStrCmp(pstrID, pszTypeExpected, strlen((char*)pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ + if(rsCStrSzStrCmp(pstrID, pszTypeExpected, ustrlen(pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ ABORT_FINALIZE(RS_RET_INVALID_OID); CHKiRet(FindObjInfo(pstrID, &pObjInfo)); @@ -949,13 +947,8 @@ SetName(obj_t *pThis, uchar *pszName) { DEFiRet; - if(pThis->pszName != NULL) - free(pThis->pszName); - - pThis->pszName = (uchar*) strdup((char*) pszName); - - if(pThis->pszName == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); finalize_it: RETiRet; @@ -1058,7 +1051,7 @@ RegisterObj(uchar *pszObjName, objInfo_t *pInfo) i = 0; while(!bFound && i < OBJ_NUM_IDS && arrObjInfo[i] != NULL) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } @@ -1097,7 +1090,7 @@ UnregisterObj(uchar *pszObjName) i = 0; while(!bFound && i < OBJ_NUM_IDS) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } diff --git a/runtime/rule.c b/runtime/rule.c index fa56dc3b..c157242c 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -317,9 +317,6 @@ CODESTARTobjDestruct(rule) } llDestroy(&pThis->llActList); - free(pThis); - - return RS_RET_OK; ENDobjDestruct(rule) diff --git a/runtime/wti.c b/runtime/wti.c index 544bffa7..18767ea1 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -201,8 +201,7 @@ CODESTARTobjDestruct(wti) pthread_cond_destroy(&pThis->condExitDone); pthread_mutex_destroy(&pThis->mut); - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->pszDbgHdr); ENDobjDestruct(wti) diff --git a/runtime/wtp.c b/runtime/wtp.c index 04eb974f..df39daa3 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -151,8 +151,7 @@ CODESTARTobjDestruct(wtp) pthread_mutex_destroy(&pThis->mut); pthread_mutex_destroy(&pThis->mutThrdShutdwn); - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->pszDbgHdr); ENDobjDestruct(wtp) -- cgit From 6141845f7514622f77d308b7aadb15891d3a627a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 18:24:46 +0200 Subject: re-enabled queue disk fsync test ... actually, it was not broken, but just very slow. I have now reduced the number of test messages so that make check will not be held for an extended period of time. --- runtime/ruleset.c | 1 - tests/diskqueue-fsync.sh | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/runtime/ruleset.c b/runtime/ruleset.c index d135074f..a1454275 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -88,7 +88,6 @@ iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void { iterateAllActions_t params; DEFiRet; - ISOBJ_TYPE_assert(pThis, ruleset); assert(pFunc != NULL); params.pFunc = pFunc; diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh index 8d2cb709..45fb864c 100755 --- a/tests/diskqueue-fsync.sh +++ b/tests/diskqueue-fsync.sh @@ -5,13 +5,11 @@ # added 2009-06-09 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -echo diskqueue-fsync test seems to have some issues -exit 1 echo testing queue disk-only mode, fsync case source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue-fsync.conf -# 5000 messages should be enough - the disk fsync test is very slow! -source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 5000 +# 1000 messages should be enough - the disk fsync test is very slow! +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 1000 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages -source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh seq-check 0 999 source $srcdir/diag.sh exit -- cgit From 1c8fe77b78a64d69138b30ec28b430677b197601 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 19:03:38 +0200 Subject: added $Ruleset config command so we now can define multiple rule sets, we just can not use them ;) That means we have the foundation to bind listeners to different rule sets. --- doc/rsyslog_conf_global.html | 2 ++ runtime/conf.c | 6 +++--- runtime/rule.c | 22 ++++++++++++++++++++++ runtime/rule.h | 3 +++ tools/syslogd.c | 27 +++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 1e268f4b..e9b1c082 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -201,6 +201,8 @@ line is that n is large enough to get a good idea which message was repeated but large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.

  • $RepeatedMsgReduction
  • $ResetConfigVariables
    • +
  • $Ruleset name - starts a new ruleset. All following actions belong to +that new rule set.
  • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common. [available since 4.1.0]
    • diff --git a/runtime/conf.c b/runtime/conf.c index e0ed3d6a..014d5a9a 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -471,7 +471,7 @@ processConfFile(uchar *pConfFile) /* we probably have one selector left to be added - so let's do that now */ if(pCurrRule != NULL) { - CHKiRet(ruleset.AddRule(pCurrRuleset, &pCurrRule)); + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(pCurrRule), &pCurrRule)); } /* close the configuration file */ @@ -1141,9 +1141,10 @@ cflineClassic(uchar *p, rule_t **ppRule) * all. -- rgerhards, 2007-08-01 */ if(*ppRule != NULL) { - CHKiRet(ruleset.AddRule(pCurrRuleset, ppRule)); + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(*ppRule), ppRule)); } CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ + CHKiRet(rule.SetAssRuleset(*ppRule, pCurrRuleset)); /* create "fresh" selector */ CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } @@ -1166,7 +1167,6 @@ cfline(uchar *line, rule_t **pfCurr) DEFiRet; ASSERT(line != NULL); -if(*pfCurr != NULL){ ISOBJ_TYPE_assert(*pfCurr, rule);} dbgprintf("cfline: '%s'\n", line); diff --git a/runtime/rule.c b/runtime/rule.c index c157242c..f17c524e 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -320,6 +320,26 @@ CODESTARTobjDestruct(rule) ENDobjDestruct(rule) +/* set the associated ruleset */ +static rsRetVal +setAssRuleset(rule_t *pThis, ruleset_t *pRuleset) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + ISOBJ_TYPE_assert(pRuleset, ruleset); + pThis->pRuleset = pRuleset; + RETiRet; +} + +/* get the associated ruleset (may be NULL if not set!) */ +static ruleset_t* +getAssRuleset(rule_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, rule); + return pThis->pRuleset; +} + + /* helper to DebugPrint, to print out all actions via * the llExecFunc() facility. */ @@ -393,6 +413,8 @@ CODESTARTobjQueryInterface(rule) pIf->IterateAllActions = iterateAllActions; pIf->ProcessMsg = processMsg; + pIf->SetAssRuleset = setAssRuleset; + pIf->GetAssRuleset = getAssRuleset; finalize_it: ENDobjQueryInterface(rule) diff --git a/runtime/rule.h b/runtime/rule.h index 96be9e30..38b11c63 100644 --- a/runtime/rule.h +++ b/runtime/rule.h @@ -53,6 +53,7 @@ struct rule_s { expr_t *f_expr; /* expression object */ } f_filterData; + ruleset_t *pRuleset; /* associated ruleset */ linkedList_t llActList; /* list of configured actions */ }; @@ -64,6 +65,8 @@ BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Destruct)(rule_t **ppThis); rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg); + rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*); + ruleset_t* (*GetAssRuleset)(rule_t *pThis); ENDinterface(rule) #define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 183c7760..f4b59970 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2398,6 +2398,32 @@ finalize_it: } +/* Begin a new rule set. The new rule set is created, and all rules that now + * follow go into that rule set. + * TODO: we may later add the capability to switch back to an already existing + * rule set. + * NOTE: pCurrRuleset is NOT desructed and must not be! The ruleset class keeps + * a list of all known rule sets, and can destruct them at the end of execution. + * pCurrRuleset is just a shortcut so that "everyone" knows which ruleset to + * extend. + * TODO: A problem with this function is the way config lines are processed. The rule + * is actually only written when the next rule is completely read. That way, this + * (past) rule goes into the wrong (new) ruleset. I need to see how to fix this best... + * rgerhards, 2009-06-10 + */ +static rsRetVal beginNewRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + DEFiRet; + CHKiRet(ruleset.Construct(&pCurrRuleset)); + CHKiRet(ruleset.SetName(pCurrRuleset, pszName)); + CHKiRet(ruleset.ConstructFinalize(pCurrRuleset)); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + /* set the main message queue mode * rgerhards, 2008-01-03 */ @@ -2651,6 +2677,7 @@ static rsRetVal loadBuildInModules(void) * This, I think, is the right thing to do. -- rgerhards, 2007-07-31 */ CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeretrycount", 0, eCmdHdlrInt, NULL, &glbliActionResumeRetryCount, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, beginNewRuleset, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszMainMsgQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesize", 0, eCmdHdlrInt, NULL, &iMainMsgQueueSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iMainMsgQHighWtrMark, NULL)); -- cgit From 0917edf26da9055c6dc160aafca97896daea3e6c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 09:47:44 +0200 Subject: re-enabled outchannel functionality --- runtime/rsyslog.h | 3 ++ runtime/srUtils.h | 1 + runtime/srutils.c | 27 ++++++++++ runtime/stream.c | 130 ++++++++++++++++++++++++++++++++++++++++++++--- runtime/stream.h | 7 ++- runtime/unicode-helper.h | 27 ++++++---- tools/omfile.c | 91 +++++++-------------------------- 7 files changed, 198 insertions(+), 88 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index b587554e..92942c70 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -289,6 +289,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */ RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */ + RS_RET_NON_SIZELIMITCMD = -2125, /**< size limit for file defined, but no size limit command given */ + RS_RET_SIZELIMITCMD_DIDNT_RESOLVE = -2126, /**< size limit command did not resolve situation */ + RS_RET_STREAM_DISABLED = -2127, /**< a file has been disabled (e.g. by size limit restriction) */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ RS_RET_ZLIB_ERR = -2141, /**< error during zlib call */ diff --git a/runtime/srUtils.h b/runtime/srUtils.h index bfce4cbb..699f8527 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -92,6 +92,7 @@ void srSleep(int iSeconds, int iuSeconds); char *rs_strerror_r(int errnum, char *buf, size_t buflen); int decodeSyslogName(uchar *name, syslogName_t *codetab); int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); +rsRetVal getFileSize(uchar *pszName, off_t *pSize); /* mutex operations */ /* some macros to cancel-safe lock a mutex (it will automatically be released diff --git a/runtime/srutils.c b/runtime/srutils.c index d01ca20d..5407531f 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -553,6 +553,33 @@ int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep) } +/* get the size of a file or return appropriate error code. If an error is returned, + * *pSize content is undefined. + * rgerhards, 2009-06-12 + */ +rsRetVal +getFileSize(uchar *pszName, off_t *pSize) +{ + int ret; + struct stat statBuf; + DEFiRet; + + ret = stat((char*) pszName, &statBuf); + if(ret == -1) { + switch(errno) { + case EACCES: ABORT_FINALIZE(RS_RET_NO_FILE_ACCESS); + case ENOTDIR: + case ENOENT: ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + default: ABORT_FINALIZE(RS_RET_FILE_NO_STAT); + } + } + + *pSize = statBuf.st_size; + +finalize_it: + RETiRet; +} + /* vim:set ai: */ diff --git a/runtime/stream.c b/runtime/stream.c index 49d29e0e..5355a1a5 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -6,8 +6,9 @@ * "driver"). * * File begun on 2008-01-09 by RGerhards + * Large modifications in 2009-06 to support using it with omfile, including zip writer. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -54,11 +55,112 @@ DEFobjCurrIf(zlibw) /* forward definitions */ static rsRetVal strmFlush(strm_t *pThis); static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal strmCloseFile(strm_t *pThis); /* methods */ -/* first, we define type-specific handlers. The provide a generic functionality, + +/* Try to resolve a size limit situation. This is used to support custom-file size handlers + * for omfile. It first runs the command, and then checks if we are still above the size + * treshold. Note that this works only with single file names, NOT with circular names. + * Note that pszCurrFName can NOT be taken from pThis, because the stream is closed when + * we are called (and that destroys pszCurrFName, as there is NO CURRENT file name!). So + * we need to receive the name as a parameter. + * initially wirtten 2005-06-21, moved to this class & updates 2009-06-01, both rgerhards + */ +static rsRetVal +resolveFileSizeLimit(strm_t *pThis, uchar *pszCurrFName) +{ + uchar *pParams; + uchar *pCmd; + uchar *p; + off_t actualFileSize; + rsRetVal localRet; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + assert(pszCurrFName != NULL); + + if(pThis->pszSizeLimitCmd == NULL) { + ABORT_FINALIZE(RS_RET_NON_SIZELIMITCMD); /* nothing we can do in this case... */ + } + + /* we first check if we have command line parameters. We assume this, + * when we have a space in the program name. If we find it, everything after + * the space is treated as a single argument. + */ + CHKmalloc(pCmd = ustrdup(pThis->pszSizeLimitCmd)); + + for(p = pCmd ; *p && *p != ' ' ; ++p) { + /* JUST SKIP */ + } + + if(*p == ' ') { + *p = '\0'; /* pretend string-end */ + pParams = p+1; + } else + pParams = NULL; + + /* the execProg() below is probably not great, but at least is is + * fairly secure now. Once we change the way file size limits are + * handled, we should also revisit how this command is run (and + * with which parameters). rgerhards, 2007-07-20 + */ + execProg(pCmd, 1, pParams); + + free(pCmd); + + localRet = getFileSize(pszCurrFName, &actualFileSize); + + if(localRet == RS_RET_OK && actualFileSize >= pThis->iSizeLimit) { + ABORT_FINALIZE(RS_RET_SIZELIMITCMD_DIDNT_RESOLVE); /* OK, it didn't work out... */ + } else if(localRet != RS_RET_FILE_NOT_FOUND) { + /* file not found is OK, the command may have moved away the file */ + ABORT_FINALIZE(localRet); + } + +finalize_it: + if(iRet != RS_RET_OK) { + if(iRet == RS_RET_SIZELIMITCMD_DIDNT_RESOLVE) + dbgprintf("file size limit cmd for file '%s' did no resolve situation\n", pszCurrFName); + else + dbgprintf("file size limit cmd for file '%s' failed with code %d.\n", pszCurrFName, iRet); + pThis->bDisabled = 1; + } + + RETiRet; +} + + +/* Check if the file has grown beyond the configured omfile iSizeLimit + * and, if so, initiate processing. + */ +static rsRetVal +doSizeLimitProcessing(strm_t *pThis) +{ + uchar *pszCurrFName = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + ASSERT(pThis->iSizeLimit != 0); + ASSERT(pThis->fd != -1); + + if(pThis->iCurrOffs >= pThis->iSizeLimit) { + /* strmClosefile() destroys the current file name, so we + * need to preserve it. + */ + CHKmalloc(pszCurrFName = ustrdup(pThis->pszCurrFName)); + CHKiRet(strmCloseFile(pThis)); + CHKiRet(resolveFileSizeLimit(pThis, pszCurrFName)); + } + +finalize_it: + free(pszCurrFName); + RETiRet; +} + + +/* now, we define type-specific handlers. The provide a generic functionality, * but for this specific type of strm. The mapping to these handlers happens during * strm construction. Later on, handlers are called by pointers present in the * strm instance object. @@ -123,6 +225,12 @@ static rsRetVal strmOpenFile(strm_t *pThis) } pThis->iCurrOffs = 0; + if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) { + /* we need to obtain the current offset */ + off_t offset; + CHKiRet(getFileSize(pThis->pszCurrFName, &offset)); + pThis->iCurrOffs = offset; + } dbgoprint((obj_t*) pThis, "opened file '%s' for %s (0x%x) as %d\n", pThis->pszCurrFName, (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", iFlags, pThis->fd); @@ -527,7 +635,7 @@ doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) if(err == EINTR) { /*NO ERROR, just continue */; } else { - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_IO_ERROR); // TODO: cover more error cases! } } @@ -538,7 +646,7 @@ doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) } while(lenBuf > 0); /* Warning: do..while()! */ finalize_it: - *pLenBuf -= iTotalWritten; + *pLenBuf = iTotalWritten; RETiRet; } @@ -610,8 +718,11 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) CHKiRet(syncFile(pThis)); } - if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { CHKiRet(strmCheckNextOutputFile(pThis)); + } else if(pThis->iSizeLimit != 0) { + CHKiRet(doSizeLimitProcessing(pThis)); + } finalize_it: pThis->iBufPtr = 0; /* see comment above */ @@ -812,7 +923,10 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) ASSERT(pThis != NULL); ASSERT(pBuf != NULL); -dbgprintf("strmWrite(%p, '%s', %ld);\n", pThis, pBuf,lenBuf); +dbgprintf("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n", pThis, pBuf,lenBuf, pThis->bDisabled, pThis->iSizeLimit, pThis->iCurrOffs); + if(pThis->bDisabled) + ABORT_FINALIZE(RS_RET_STREAM_DISABLED); + /* check if the to-be-written data is larger than our buffer size */ if(lenBuf >= pThis->sIOBufSize) { /* it is - so we do a direct write, that is most efficient. @@ -857,6 +971,8 @@ DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) +DEFpropSetMeth(strm, iSizeLimit, off_t) +DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { @@ -1139,6 +1255,8 @@ CODESTARTobjQueryInterface(strm) pIf->SetiZipLevel = strmSetiZipLevel; pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; + pIf->SetiSizeLimit = strmSetiSizeLimit; + pIf->SetpszSizeLimitCmd = strmSetpszSizeLimitCmd; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index a66108b7..a06e089f 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -103,6 +103,7 @@ typedef struct strm_s { int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ + int bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ int bSync; /* sync this file after every write? */ size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ @@ -117,7 +118,9 @@ typedef struct strm_s { int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; - + /* support for omfile size-limiting commands, special counters, NOT persisted! */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ } strm_t; /* interfaces */ @@ -152,6 +155,8 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, iZipLevel, int); INTERFACEpropSetMeth(strm, bSync, int); INTERFACEpropSetMeth(strm, sIOBufSize, size_t); + INTERFACEpropSetMeth(strm, iSizeLimit, off_t); + INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 5289da3f..8216e992 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -4,6 +4,9 @@ * The following functions are wrappers which hopefully enable us to move * from 8-bit chars to unicode with relative ease when we finally attack this * + * Note: while we prefer inline functions, this leads to invalid references in + * core dumps. So in a debug build, we use macros where appropriate... + * * Begun 2009-05-21 RGerhards * * Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH @@ -31,10 +34,21 @@ #include -static inline char* ustrncpy(uchar *psz1, uchar *psz2, size_t len) -{ - return strncpy((char*) psz1, (char*) psz2, len); -} +#ifdef DEBUG +# define ustrncpy(psz1, psz2, len) strncpy((char*)(psz1), (char*)(psz2), (len)) +# define ustrdup(psz) (uchar*)strdup((char*)(psz)) +#else + static inline uchar* ustrncpy(uchar *psz1, uchar *psz2, size_t len) + { + return strncpy((char*) psz1, (char*) psz2, len); + } + + static inline uchar* ustrdup(uchar *psz) + { + return (uchar*) strdup((char*)psz); + } + +#endif /* #ifdef DEBUG */ static inline int ustrcmp(uchar *psz1, uchar *psz2) { @@ -46,11 +60,6 @@ static inline int ustrlen(uchar *psz) return strlen((char*) psz); } -static inline uchar* ustrdup(uchar *psz) -{ - return (uchar*) strdup((char*)psz); -} - #define UCHAR_CONSTANT(x) ((uchar*) (x)) #define CHAR_CONVERT(x) ((char*) (x)) diff --git a/tools/omfile.c b/tools/omfile.c index e3f7af19..cf020f7b 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -67,6 +67,7 @@ #include "unicode-helper.h" #include "stream.h" #include "zlibw.h" +#include "unicode-helper.h" MODULE_TYPE_OUTPUT @@ -119,7 +120,7 @@ typedef struct _instanceData { int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ - int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! + int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -134,7 +135,7 @@ typedef struct _instanceData { */ dynaFileCacheEntry **dynCache; off_t iSizeLimit; /* file size limit, 0 = no limit */ - uchar *iSizeLimitCmd; /* command to carry out when size limit is reached */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ } instanceData; @@ -264,7 +265,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->iSizeLimitCmd = pOch->cmdOnSizeLimit; + pData->pszSizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -274,66 +275,6 @@ finalize_it: } -#if 0 -/* rgerhards 2005-06-21: Try to resolve a size limit - * situation. This first runs the command, and then - * checks if we are still above the treshold. - * returns 0 if ok, 1 otherwise - * TODO: consider moving the initial check in here, too - */ -int resolveFileSizeLimit(instanceData *pData) -{ - uchar *pParams; - uchar *pCmd; - uchar *p; - off_t actualFileSize; - ASSERT(pData != NULL); - - if(pData->iSizeLimitCmd == NULL) - return 1; /* nothing we can do in this case... */ - - /* the execProg() below is probably not great, but at least is is - * fairly secure now. Once we change the way file size limits are - * handled, we should also revisit how this command is run (and - * with which parameters). rgerhards, 2007-07-20 - */ - /* we first check if we have command line parameters. We assume this, - * when we have a space in the program name. If we find it, everything after - * the space is treated as a single argument. - */ - if((pCmd = ustrdup(pData->iSizeLimitCmd)) == NULL) { - /* there is not much we can do - we make syslogd close the file in this case */ - return 1; - } - - for(p = pCmd ; *p && *p != ' ' ; ++p) { - /* JUST SKIP */ - } - - if(*p == ' ') { - *p = '\0'; /* pretend string-end */ - pParams = p+1; - } else - pParams = NULL; - - execProg(pCmd, 1, pParams); - - free(pCmd); - - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - - actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->iSizeLimit) { - /* OK, it didn't work out... */ - return 1; - } - - return 0; -} -#endif - - /* This function deletes an entry from the dynamic file name * cache. A pointer to the cache must be passed in as well * as the index of the to-be-deleted entry. This index may @@ -458,23 +399,29 @@ prepareFile(instanceData *pData, uchar *newFileName) } } - char szNameBuf[MAXFNAME]; - char szDirName[MAXFNAME]; - char szBaseName[MAXFNAME]; - strcpy(szNameBuf, (char*)newFileName); - strcpy(szDirName, dirname(szNameBuf)); - strcpy(szNameBuf, (char*)newFileName); - strcpy(szBaseName, basename(szNameBuf)); + /* the copies below are clumpsy, but there is no way around given the + * anomalies in dirname() and basename() [they MODIFY the provided buffer...] + */ + uchar szNameBuf[MAXFNAME]; + uchar szDirName[MAXFNAME]; + uchar szBaseName[MAXFNAME]; + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szDirName, (uchar*)dirname((char*)szNameBuf), MAXFNAME); + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szBaseName, (uchar*)basename((char*)szNameBuf), MAXFNAME); CHKiRet(strm.Construct(&pData->pStrm)); - CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); - CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); + CHKiRet(strm.SetFName(pData->pStrm, szBaseName, ustrlen(szBaseName))); + CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetiSizeLimit(pData->pStrm, pData->iSizeLimit)); + if(pData->pszSizeLimitCmd != NULL) + CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd))); CHKiRet(strm.ConstructFinalize(pData->pStrm)); finalize_it: -- cgit From 7f7e5ef75a6183c3b625afe58984343f9b7997be Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 09:57:50 +0200 Subject: improved config error messages now contain a copy of the config line that (most likely) caused the error --- ChangeLog | 2 ++ runtime/conf.c | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 57bb6e7e..39ca00ec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- improved config error messages: now contain a copy of the config line + that (most likely) caused the error - added a generic network stream server (in addition to rather specific syslog tcp server) - added ability for the UDP output action to rebind its send socket after diff --git a/runtime/conf.c b/runtime/conf.c index 014d5a9a..412a756d 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -71,6 +71,7 @@ #include "ctok_token.h" #include "rule.h" #include "ruleset.h" +#include "unicode-helper.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN @@ -396,7 +397,6 @@ finalize_it: static rsRetVal processConfFile(uchar *pConfFile) { - DEFiRet; int iLnNbr = 0; FILE *cf; rule_t *pCurrRule = NULL; @@ -405,6 +405,9 @@ processConfFile(uchar *pConfFile) uchar *cline; int i; int bHadAnError = 0; + uchar *pszOrgLine = NULL; + size_t lenLine; + DEFiRet; ASSERT(pConfFile != NULL); if((cf = fopen((char*)pConfFile, "r")) == NULL) { @@ -417,9 +420,12 @@ processConfFile(uchar *pConfFile) while (fgets((char*)cline, sizeof(cbuf) - (cline - cbuf), cf) != NULL) { ++iLnNbr; /* drop LF - TODO: make it better, replace fgets(), but its clean as it is */ - if(cline[strlen((char*)cline)-1] == '\n') { - cline[strlen((char*)cline) -1] = '\0'; + lenLine = ustrlen(cline); + if(cline[lenLine-1] == '\n') { + cline[lenLine-1] = '\0'; } + free(pszOrgLine); + pszOrgLine = ustrdup(cline); /* save if needed for errmsg, NULL ptr is OK */ /* check for end-of-section, comments, strip off trailing * spaces and newline character. */ @@ -464,7 +470,7 @@ processConfFile(uchar *pConfFile) dbgprintf("config line NOT successfully processed\n"); snprintf((char*)szErrLoc, sizeof(szErrLoc) / sizeof(uchar), "%s, line %d", pConfFile, iLnNbr); - errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s", (char*)szErrLoc); + errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s:\"%s\"", (char*)szErrLoc, (char*)pszOrgLine); bHadAnError = 1; } } @@ -488,6 +494,8 @@ finalize_it: iRet, pConfFile, errStr); } + free(pszOrgLine); + if(bHadAnError && (iRet == RS_RET_OK)) { /* a bit dirty, enhance in future releases */ iRet = RS_RET_NONFATAL_CONFIG_ERR; } -- cgit From 1af948107e6e520788e374adccf4986bf07e92f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 10:14:45 +0200 Subject: fixed abort when emergency configuration was activated this regression was introduced last friday, so this is *NOT* present in any released version. --- runtime/conf.c | 1 - tools/syslogd.c | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/runtime/conf.c b/runtime/conf.c index 412a756d..81b6c081 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -1193,7 +1193,6 @@ cfline(uchar *line, rule_t **pfCurr) break; default: iRet = cflineClassic(line, pfCurr); -ISOBJ_TYPE_assert(*pfCurr, rule); break; } diff --git a/tools/syslogd.c b/tools/syslogd.c index f4b59970..96f3cb34 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2247,12 +2247,11 @@ init(void) * too low on linux... :-S -- rgerhards, 2008-07-28 */ char szTTYNameBuf[128]; - rule_t *pRule; - CHKiRet(rule.Construct(&pRule)); - conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &pRule); - conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &pRule); - conf.cfline((uchar*)"*.PANIC\t*", &pRule); - conf.cfline((uchar*)"syslog.*\troot", &pRule); + rule_t *pRule = NULL; /* initialization to NULL is *vitally* important! */ + conf.cfline(UCHAR_CONSTANT("*.ERR\t" _PATH_CONSOLE), &pRule); + conf.cfline(UCHAR_CONSTANT("syslog.*\t" _PATH_CONSOLE), &pRule); + conf.cfline(UCHAR_CONSTANT("*.PANIC\t*"), &pRule); + conf.cfline(UCHAR_CONSTANT("syslog.*\troot"), &pRule); if(ttyname_r(0, szTTYNameBuf, sizeof(szTTYNameBuf)) == 0) { snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); conf.cfline((uchar*)cbuf, &pRule); -- cgit From e3d9843c85b1dfddabc937ac6ccb4057d626bf03 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 11:47:00 +0200 Subject: re-enabled pipe, tty and console in omfile ... by moving code to stream.c. Thanks to the new design, new cases are not really needed, resulting in cleaner code. I also did a cleanup of header file usage as a side-activity. --- dirty.h | 4 - plugins/omgssapi/omgssapi.c | 1 + plugins/ommail/ommail.c | 2 +- plugins/ommysql/ommysql.c | 2 +- plugins/ompgsql/ompgsql.c | 2 +- plugins/omprog/omprog.c | 2 +- plugins/omrelp/omrelp.c | 2 +- plugins/omsnmp/omsnmp.c | 2 +- plugins/omstdout/omstdout.c | 2 +- plugins/omtemplate/omtemplate.c | 2 +- runtime/conf.h | 4 + runtime/rsyslog.h | 13 ++++ runtime/stream.c | 162 ++++++++++++++++++++++++++-------------- runtime/stream.h | 5 +- tools/omfile.c | 142 +---------------------------------- tools/omfwd.c | 1 + tools/omshell.c | 2 +- tools/omusrmsg.c | 2 +- tools/syslogd.c | 11 +-- tools/syslogd.h | 10 --- 20 files changed, 147 insertions(+), 226 deletions(-) diff --git a/dirty.h b/dirty.h index bab15485..7ae57d09 100644 --- a/dirty.h +++ b/dirty.h @@ -35,10 +35,6 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ char* getFIOPName(unsigned iFIOP); -/* TODO: the following 2 need to go in conf obj interface... */ -rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); -rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); - /* Intervals at which we flush out "message repeated" messages, * in seconds after previous message is logged. After each flush, * we move to the next interval until we reach the largest. diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 361f657f..7b5a46e1 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -44,6 +44,7 @@ #include #include #include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "net.h" diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index 5faadce3..3a7669c9 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -44,7 +44,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "cfsysline.h" diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index ecf738a9..d6870a7b 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 6daac1c7..eb774835 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -40,7 +40,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 2a078a6d..01fa7cea 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 8d74c82f..d5ef8b4f 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "cfsysline.h" diff --git a/plugins/omsnmp/omsnmp.c b/plugins/omsnmp/omsnmp.c index 72fa8d64..4db60e62 100644 --- a/plugins/omsnmp/omsnmp.c +++ b/plugins/omsnmp/omsnmp.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "cfsysline.h" #include "module-template.h" diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 181895a4..b9125f19 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -35,7 +35,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omtemplate/omtemplate.c b/plugins/omtemplate/omtemplate.c index e35968ad..5577f8c6 100644 --- a/plugins/omtemplate/omtemplate.c +++ b/plugins/omtemplate/omtemplate.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/runtime/conf.h b/runtime/conf.h index f29de1ba..25b887be 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -51,5 +51,9 @@ PROTOTYPEObj(conf); extern EHostnameCmpMode eDfltHostnameCmpMode; extern cstr_t *pDfltHostnameCmp; extern cstr_t *pDfltProgNameCmp; +/* TODO: the following 2 need to go in conf obj interface... */ +rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); +rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); + #endif /* #ifndef INCLUDED_CONF_H */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 92942c70..28f7bce9 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -99,6 +99,7 @@ typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef short bool; #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ @@ -124,6 +125,18 @@ typedef enum { FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } fiop_t; +/* file types (omfile & stream) */ +typedef enum { + eTypeFILE, + eTypeTTY, + eTypeCONSOLE, + eTypePIPE +} filetype_t; + + +#ifndef _PATH_CONSOLE +#define _PATH_CONSOLE "/dev/console" +#endif /* The error codes below are orginally "borrowed" from * liblogging. As such, we reserve values up to -2999 diff --git a/runtime/stream.c b/runtime/stream.c index 5355a1a5..c8672aa2 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -166,35 +166,14 @@ finalize_it: * strm instance object. */ -/* open a strm file - * It is OK to call this function when the stream is already open. In that - * case, it returns immediately with RS_RET_OK +/* do the physical open() call on a file. */ -static rsRetVal strmOpenFile(strm_t *pThis) +static rsRetVal +doPhysOpen(strm_t *pThis) { - DEFiRet; int iFlags; - - ASSERT(pThis != NULL); - - if(pThis->fd != -1) - ABORT_FINALIZE(RS_RET_OK); - - if(pThis->pszFName == NULL) - ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - - if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { - CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, - pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); - } else { - if(pThis->pszDir == NULL) { - if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } else { - CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, - pThis->pszFName, pThis->lenFName, -1, 0)); - } - } + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); /* compute which flags we need to provide to open */ switch(pThis->tOperationsMode) { @@ -222,8 +201,51 @@ static rsRetVal strmOpenFile(strm_t *pThis) ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); else ABORT_FINALIZE(RS_RET_IO_ERROR); + } else { + if(!ustrcmp(pThis->pszCurrFName, UCHAR_CONSTANT(_PATH_CONSOLE)) || isatty(pThis->fd)) { + DBGPRINTF("file %d is a tty-type file\n", pThis->fd); + pThis->bIsTTY = 1; + } else { + pThis->bIsTTY = 0; + } } +finalize_it: + RETiRet; +} + + +/* open a strm file + * It is OK to call this function when the stream is already open. In that + * case, it returns immediately with RS_RET_OK + */ +static rsRetVal strmOpenFile(strm_t *pThis) +{ + DEFiRet; + + ASSERT(pThis != NULL); + + if(pThis->fd != -1) + ABORT_FINALIZE(RS_RET_OK); + + if(pThis->pszFName == NULL) + ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); + + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); + } else { + if(pThis->pszDir == NULL) { + if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } else { + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, -1, 0)); + } + } + + CHKiRet(doPhysOpen(pThis)); + pThis->iCurrOffs = 0; if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) { /* we need to obtain the current offset */ @@ -232,8 +254,8 @@ static rsRetVal strmOpenFile(strm_t *pThis) pThis->iCurrOffs = offset; } - dbgoprint((obj_t*) pThis, "opened file '%s' for %s (0x%x) as %d\n", pThis->pszCurrFName, - (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", iFlags, pThis->fd); + dbgoprint((obj_t*) pThis, "opened file '%s' for %s as %d\n", pThis->pszCurrFName, + (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", pThis->fd); finalize_it: RETiRet; @@ -255,7 +277,7 @@ static rsRetVal strmCloseFile(strm_t *pThis) if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); - close(pThis->fd); // TODO: error check + close(pThis->fd); pThis->fd = -1; if(pThis->fdDir != -1) { @@ -265,7 +287,13 @@ static rsRetVal strmCloseFile(strm_t *pThis) } if(pThis->bDeleteOnClose) { - unlink((char*) pThis->pszCurrFName); // TODO: check returncode + if(unlink((char*) pThis->pszCurrFName) == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("error %d unlinking '%s' - ignored: %s\n", + errno, pThis->pszCurrFName, errStr); + } } pThis->iCurrOffs = 0; /* we are back at begin of file */ @@ -544,13 +572,12 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } /* if we are aset to sync, we must obtain a file handle to the directory for fsync() purposes */ - if(pThis->bSync) { - pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY); + if(pThis->bSync && !pThis->bIsTTY) { + pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY | O_CLOEXEC | O_NOCTTY); if(pThis->fdDir == -1) { char errStr[1024]; int err = errno; rs_strerror_r(err, errStr, sizeof(errStr)); - // TODO: log an error message? think so... DBGPRINTF("error %d opening directory file for fsync() use - fsync for directory disabled: %s\n", errno, errStr); } @@ -606,6 +633,29 @@ finalize_it: } +/* try to recover a tty after a write error. This may have happend + * due to vhangup(), and, if so, we can simply re-open it. + */ +#ifdef linux +# define ERR_TTYHUP EIO +#else +# define ERR_TTYHUP EBADF +#endif +static rsRetVal +tryTTYRecover(strm_t *pThis, int err) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + if(err == ERR_TTYHUP) { + close(pThis->fd); + CHKiRet(doPhysOpen(pThis)); + } + +finalize_it: + RETiRet; +} +#undef ER_TTYHUP + /* issue write() api calls until either the buffer is completely * written or an error occured (it may happen that multiple writes @@ -614,29 +664,36 @@ finalize_it: * rgerhards, 2009-06-08 */ static rsRetVal -doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) +doWriteCall(strm_t *pThis, uchar *pBuf, size_t *pLenBuf) { ssize_t lenBuf; ssize_t iTotalWritten; ssize_t iWritten; char *pWriteBuf; DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); lenBuf = *pLenBuf; pWriteBuf = (char*) pBuf; iTotalWritten = 0; do { - iWritten = write(fd, pWriteBuf, lenBuf); + iWritten = write(pThis->fd, pWriteBuf, lenBuf); if(iWritten < 0) { char errStr[1024]; int err = errno; rs_strerror_r(err, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", fd, err, errStr); + DBGPRINTF("log file (%d) write error %d: %s\n", pThis->fd, err, errStr); if(err == EINTR) { /*NO ERROR, just continue */; } else { - ABORT_FINALIZE(RS_RET_IO_ERROR); - // TODO: cover more error cases! + if(pThis->bIsTTY) { + CHKiRet(tryTTYRecover(pThis, err)); + } else { + ABORT_FINALIZE(RS_RET_IO_ERROR); + /* Would it make sense to cover more error cases? So far, I + * do not see good reason to do so. + */ + } } } /* advance buffer to next write position */ @@ -653,7 +710,10 @@ finalize_it: /* sync the file to disk, so that any unwritten data is persisted. This * also syncs the directory and thus makes sure that the file survives - * fatal failure. -- rgerhards, 2009-06-08 + * fatal failure. Note that we do NOT return an error status if the + * sync fails. Doing so would probably cause more trouble than it + * is worth (read: data loss may occur where we otherwise might not + * have it). -- rgerhards, 2009-06-08 */ static rsRetVal syncFile(strm_t *pThis) @@ -661,6 +721,9 @@ syncFile(strm_t *pThis) int ret; DEFiRet; + if(pThis->bIsTTY) + FINALIZE; /* TTYs can not be synced */ + DBGPRINTF("syncing file %d\n", pThis->fd); ret = fdatasync(pThis->fd); if(ret != 0) { @@ -670,19 +733,20 @@ syncFile(strm_t *pThis) DBGPRINTF("sync failed for file %d with error (%d): %s - ignoring\n", pThis->fd, err, errStr); } - // TODO: check error! if(pThis->fdDir != -1) { ret = fsync(pThis->fdDir); -dbgprintf("sync on dir (fd %d) requested, return code %d\n", pThis->fdDir, ret); } +finalize_it: RETiRet; } /* physically write to the output file. the provided data is ready for * writing (e.g. zipped if we are requested to do that). + * Note that if the write() API fails, we do not reset any pointers, but return + * an error code. That means we may redo work in the next iteration. * rgerhards, 2009-06-04 */ static rsRetVal @@ -690,24 +754,15 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { size_t iWritten; DEFiRet; - - ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pThis, strm); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); iWritten = lenBuf; - CHKiRet(doWriteCall(pThis->fd, pBuf, &iWritten)); + CHKiRet(doWriteCall(pThis, pBuf, &iWritten)); dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, (int) iWritten); - /* Now indicate buffer empty again. We do this in any case, because there - * is no way we could react more intelligently to an error during write. - * This MUST be done BEFORE strCheckNextOutputFile(), otherwise we have an - * endless loop. We reset the buffer pointer also in finalize_it - this is - * necessary if we run into problems. Not resetting it would again cause an - * endless loop. So it is better to loose some data (which also justifies - * duplicating that code, too...) -- rgerhards, 2008-01-10 - */ pThis->iBufPtr = 0; pThis->iCurrOffs += iWritten; /* update user counter, if provided */ @@ -725,8 +780,6 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) } finalize_it: - pThis->iBufPtr = 0; /* see comment above */ - RETiRet; } @@ -995,7 +1048,6 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) ASSERT(pThis != NULL); ASSERT(pszName != NULL); -dbgprintf("XXX: strm setFname: '%s'\n", pszName); if(iLenName < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); diff --git a/runtime/stream.h b/runtime/stream.h index a06e089f..021c4792 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -99,7 +99,7 @@ typedef struct strm_s { int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ - int bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ + bool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ @@ -115,12 +115,13 @@ typedef struct strm_s { size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ - int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + bool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ + bool bIsTTY; /* is this a tty file? */ } strm_t; /* interfaces */ diff --git a/tools/omfile.c b/tools/omfile.c index cf020f7b..6377268d 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -55,7 +55,7 @@ # include #endif -#include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" @@ -64,7 +64,6 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" -#include "unicode-helper.h" #include "stream.h" #include "zlibw.h" #include "unicode-helper.h" @@ -110,12 +109,6 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - enum { - eTypeFILE, - eTypeTTY, - eTypeCONSOLE, - eTypePIPE - } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ @@ -218,14 +211,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR struct outchannel *pOch; char szBuf[128]; /* should be more than sufficient */ - /* this must always be a file, because we can not set a size limit - * on a pipe... - * rgerhards 2005-06-21: later, this will be a separate type, but let's - * emulate things for the time being. When everything runs, we can - * extend it... - */ - pData->fileType = eTypeFILE; - ++p; /* skip '$' */ i = 0; /* get outchannel name */ @@ -358,8 +343,6 @@ prepareFile(instanceData *pData, uchar *newFileName) int fd; DEFiRet; - // TODO: handle TTY/PIPE case! (in stream.c!) 2009-06-04 - if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ fd = -1; @@ -425,18 +408,6 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.ConstructFinalize(pData->pStrm)); finalize_it: - /* this was "fd != 0", which I think was a bug. I guess 0 was intended to mean - * non-open file descriptor. Anyhow, I leave this comment for the time being to that if - * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 - */ -#if 0 // TODO: this must be done by stream class! - if(fd != -1 && isatty(fd)) { - DBGPRINTF("file %d is a tty file\n", fd); - pData->fileType = eTypeTTY; - untty(); - } -#endif - RETiRet; } @@ -546,107 +517,6 @@ finalize_it: } -#if 0 -// TODO: mirgrate code below to stream class (update its write handler, which is not great!) -/* physically write the file - */ -static rsRetVal -doPhysWrite(instanceData *pData, int fd, char *pszBuf, size_t lenBuf) -{ - off_t actualFileSize; - int iLenWritten; - DEFiRet; - ASSERT(pData != NULL); - -dbgprintf("doPhysWrite, fd %d, iBuf %d\n", fd, (int) lenBuf); -again: - /* check if we have a file size limit and, if so, - * obey to it. - */ - if(pData->iSizeLimit != 0) { - actualFileSize = lseek(fd, 0, SEEK_END); - if(actualFileSize >= pData->iSizeLimit) { - char errMsg[256]; - /* for now, we simply disable a file once it is - * beyond the maximum size. This is better than having - * us aborted by the OS... rgerhards 2005-06-21 - */ - (void) close(fd); - /* try to resolve the situation */ - // TODO: *doesn't work, will need to use new fd ! - if(resolveFileSizeLimit(pData) != 0) { - /* didn't work out, so disable... */ - snprintf(errMsg, sizeof(errMsg), - "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg); - ABORT_FINALIZE(RS_RET_DISABLE_ACTION); - } else { - snprintf(errMsg, sizeof(errMsg), - "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); - } - } - } - - iLenWritten = write(fd, pszBuf, lenBuf); - if(iLenWritten < 0) { - int e = errno; - char errStr[1024]; - rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", fd, e, errStr); - - /* If a named pipe is full, we suspend this action for a while */ - if(pData->fileType == eTypePIPE && e == EAGAIN) - ABORT_FINALIZE(RS_RET_SUSPENDED); - - close(pData->fd); - pData->fd = -1; /* tell that fd is no longer open! */ - if(pData->bDynamicName && pData->iCurrElt != -1) { - /* in this case, we need to invalidate the name in the cache, too - * otherwise, an invalid fd may show up if we had a file name change. - * rgerhards, 2009-03-19 - */ - pData->dynCache[pData->iCurrElt]->fd = -1; - } - /* Check for EBADF on TTY's due to vhangup() - * Linux uses EIO instead (mrn 12 May 96) - */ - if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) -#ifdef linux - && e == EIO -#else - && e == EBADF -#endif - ) { - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC); - if (pData->fd < 0) { - iRet = RS_RET_SUSPENDED; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } else { - untty(); - goto again; - } - } else { - iRet = RS_RET_SUSPENDED; - errno = e; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } - } else if (pData->bSyncFile) { - fsync(fd); - } - - pData->poBuf->iBuf = 0; - -finalize_it: - RETiRet; -} -#endif - - /* do the actual write process. This function is to be called once we are ready for writing. * It will do buffered writes and persist data only when the buffer is full. Note that we must * be careful to detect when the file handle changed. @@ -662,7 +532,7 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); - FINALIZE; // TODO: clean up later + FINALIZE; } finalize_it: @@ -780,12 +650,6 @@ CODESTARTparseSelectorAct case '|': case '/': CODE_STD_STRING_REQUESTparseSelectorAct(1) - if(*p == '|') { - pData->fileType = eTypePIPE; - ++p; - } else { - pData->fileType = eTypeFILE; - } CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); pData->bDynamicName = 0; @@ -808,8 +672,6 @@ CODESTARTparseSelectorAct pData->iIOBufSize = iIOBufSize; if(pData->bDynamicName == 0) { - if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) - pData->fileType = eTypeCONSOLE; /* try open and emit error message if not possible. At this stage, we ignore the * return value of prepareFile, this is taken care of in later steps. */ diff --git a/tools/omfwd.c b/tools/omfwd.c index c8fedfc9..2966a5e4 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -48,6 +48,7 @@ #endif #include #include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "net.h" diff --git a/tools/omshell.c b/tools/omshell.c index 7b815869..f8a68527 100644 --- a/tools/omshell.c +++ b/tools/omshell.c @@ -38,7 +38,7 @@ #include #include #include -#include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "omshell.h" diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c index 830bbc87..499a11dd 100644 --- a/tools/omusrmsg.c +++ b/tools/omusrmsg.c @@ -66,7 +66,7 @@ #include "srUtils.h" #include "stringbuf.h" #include "syslogd-types.h" -#include "syslogd.h" +#include "conf.h" #include "omusrmsg.h" #include "module-template.h" #include "errmsg.h" diff --git a/tools/syslogd.c b/tools/syslogd.c index 96f3cb34..05c61059 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -138,6 +138,7 @@ #include "unicode-helper.h" #include "ruleset.h" #include "rule.h" +#include "net.h" #include "vm.h" /* definitions for objects we access */ @@ -221,7 +222,7 @@ static rsRetVal GlobalClassExit(void); #endif #ifndef _PATH_TTY -#define _PATH_TTY "/dev/tty" +# define _PATH_TTY "/dev/tty" #endif static uchar *ConfFile = (uchar*) _PATH_LOGCONF; /* read-only after startup */ @@ -504,7 +505,7 @@ static char **crunch_list(char *list) void untty(void) #ifdef HAVE_SETSID { - if ( !Debug ) { + if(!Debug) { setsid(); } return; @@ -513,18 +514,18 @@ void untty(void) { int i; - if ( !Debug ) { + if(!Debug) { i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) - (void) ioctl(i, (int) TIOCNOTTY, (char *)0); + (void) ioctl(i, (int) TIOCNOTTY, NULL); # else /* TODO: we need to implement something for HP UX! -- rgerhards, 2008-03-04 */ /* actually, HP UX should have setsid, so the code directly above should * trigger. So the actual question is why it doesn't do that... */ # endif - (void) close(i); + close(i); } } } diff --git a/tools/syslogd.h b/tools/syslogd.h index 9bcfb2d2..3dfdbe2b 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -28,17 +28,7 @@ #include "action.h" #include "linkedlist.h" #include "expr.h" -#include "net.h" /* TODO: remove when you remove isAllowedSender from here! */ - -#ifndef _PATH_CONSOLE -#define _PATH_CONSOLE "/dev/console" -#endif - -void untty(void); -rsRetVal selectorConstruct(ruleset_t **ppThis); -rsRetVal selectorDestruct(void *pVal); -rsRetVal selectorAddList(rule_t *f); /* the following prototypes should go away once we have an input * module interface -- rgerhards, 2007-12-12 */ -- cgit From 21dafea3ee98d16a8fe93d0d5228939dc259aea7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 12:18:17 +0200 Subject: implemented $OMFileFlushOnTXEnd directive plus some cleanup... --- ChangeLog | 1 + doc/rsyslog_conf_global.html | 7 +++++++ runtime/rsyslog.h | 8 -------- runtime/ruleset.c | 9 ++++++--- tools/omfile.c | 12 +++++++++++- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 39ca00ec..86d72c9e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,7 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - added configuration commands (see doc for explanations) * $OMFileZipLevel * $OMFileIOBufferSize + * $OMFileFlushOnTXEnd * $MainMsgQueueSyncQueueFiles * $ActionQueueSyncQueueFiles --------------------------------------------------------------------------- diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index e9b1c082..a909b00c 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -193,6 +193,13 @@ supported in order to be compliant to the upcoming new syslog RFC series. of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.
  • $OMFileIOBufferSize <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)
    • +
  • $OMFileFlushOnTXEnd <[on/off]>, default off, by default, omfile +writes output using a buffered writer. Disk writes are only done when the buffer is +full. So if an error happens during that write, data is potentially lost. In cases where +this is unacceptable, set $OMFileFlushOnTXEnd to on. Then, data is written at the end +of each transaction (for pre-v5 this means after each log message) and the usual +error recovery thus can handle write errors without data loss. Note that this option +severely reduces the effect of zip compression.
  • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 28f7bce9..9421ca67 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -125,14 +125,6 @@ typedef enum { FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } fiop_t; -/* file types (omfile & stream) */ -typedef enum { - eTypeFILE, - eTypeTTY, - eTypeCONSOLE, - eTypePIPE -} filetype_t; - #ifndef _PATH_CONSOLE #define _PATH_CONSOLE "/dev/console" diff --git a/runtime/ruleset.c b/runtime/ruleset.c index a1454275..f9edde8b 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -68,7 +68,7 @@ static rsRetVal keyDestruct(void __attribute__((unused)) *pData) /* ---------- END linked-list key handling functions ---------- */ -/* dirver to iterate over all of this ruleset actions */ +/* driver to iterate over all of this ruleset actions */ typedef struct iterateAllActions_s { rsRetVal (*pFunc)(void*, void*); void *pParam; @@ -81,6 +81,7 @@ DEFFUNC_llExecFunc(doIterateRulesetActions) iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); RETiRet; } +#if 0 /* iterate over all actions of THIS rule set. */ static rsRetVal @@ -99,7 +100,7 @@ finalize_it: } -/* dirver to iterate over all actions */ +/* driver to iterate over all actions */ DEFFUNC_llExecFunc(doIterateAllActions) { DEFiRet; @@ -108,6 +109,7 @@ DEFFUNC_llExecFunc(doIterateAllActions) iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); RETiRet; } +#endif /* iterate over ALL actions present in the WHOLE system. * this is often needed, for example when HUP processing * must be done or a shutdown is pending. @@ -121,7 +123,8 @@ iterateAllActions(rsRetVal (*pFunc)(void*, void*), void* pParam) params.pFunc = pFunc; params.pParam = pParam; - CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + //CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + CHKiRet(llExecFunc(&llRulesets, doIterateRulesetActions, ¶ms)); finalize_it: RETiRet; diff --git a/tools/omfile.c b/tools/omfile.c index 6377268d..6d9eb096 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -101,6 +101,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ +static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -131,6 +132,7 @@ typedef struct _instanceData { uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ + bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; @@ -595,7 +597,12 @@ ENDtryResume BEGINdoAction CODESTARTdoAction DBGPRINTF("file to log to: %s\n", pData->f_fname); - iRet = writeFile(ppString, iMsgOpts, pData); + CHKiRet(writeFile(ppString, iMsgOpts, pData)); + if(pData->bFlushOnTXEnd) { + /* TODO v5: do this in endTransaction only! */ + CHKiRet(strm.Flush(pData->pStrm)); + } +finalize_it: ENDdoAction @@ -669,6 +676,7 @@ CODESTARTparseSelectorAct pData->dirUID = dirUID; pData->dirGID = dirGID; pData->iZipLevel = iZipLevel; + pData->bFlushOnTXEnd = bFlushOnTXEnd; pData->iIOBufSize = iIOBufSize; if(pData->bDynamicName == 0) { @@ -702,6 +710,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; + bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; if(pszTplName != NULL) { free(pszTplName); @@ -751,6 +760,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); -- cgit From ca0ddc30a3edce02a440904a01f0b866c0f82b5a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 15:31:08 +0200 Subject: completed multi-ruleset core support ... as well as added multi-ruleset support for imtcp --- dirty.h | 1 - doc/rsyslog_conf_global.html | 10 ++++-- plugins/imtcp/imtcp.c | 43 ++++++++++++++++++----- runtime/cfsysline.c | 6 ++-- runtime/conf.c | 2 +- runtime/msg.c | 13 +++++-- runtime/msg.h | 2 ++ runtime/obj.c | 4 +-- runtime/rsyslog.c | 20 +++++------ runtime/ruleset.c | 84 ++++++++++++++++++++++++++++++++++++++++++-- runtime/ruleset.h | 6 +++- tcps_sess.c | 1 + tcpsrv.c | 17 +++++++++ tcpsrv.h | 3 ++ tools/syslogd.c | 75 ++++++++++++++++++++++++--------------- 15 files changed, 226 insertions(+), 61 deletions(-) diff --git a/dirty.h b/dirty.h index 7ae57d09..513886b5 100644 --- a/dirty.h +++ b/dirty.h @@ -53,7 +53,6 @@ extern int bReduceRepeatMsgs; extern int bDropTrailingLF; extern uchar cCCEscapeChar; extern int bEscapeCCOnRcv; -extern ruleset_t *pCurrRuleset; #ifdef USE_NETZIP /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index a909b00c..d58bcac0 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -108,6 +108,9 @@ that no rebind is done. This directive is useful for use with load-balancers.$DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
  • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
  • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
    • +
  • $DefaultRuleset name - changes the default ruleset for unbound inputs to +the provided name (the default default ruleset is named +"RSYSLOG_DefaultRuleset").
  • $CreateDirs [on/off] - create directories on an as-needed basis
  • $DirCreateMode
  • $DirGroup
    • @@ -208,8 +211,11 @@ line is that n is large enough to get a good idea which message was repeated but large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.
  • $RepeatedMsgReduction
  • $ResetConfigVariables
    • -
  • $Ruleset name - starts a new ruleset. All following actions belong to -that new rule set.
    • +
  • $Ruleset name - starts a new ruleset or switches back to one already defined. +All following actions belong to that new rule set. +the name does not yet exist, it is created. To swith back to rsyslog's +default ruleset, specify "RSYSLOG_DefaultRuleset") as the name. +All following actions belong to that new rule set.
  • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common. [available since 4.1.0]
    • diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 84e660bc..e1f513c8 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -61,6 +61,7 @@ #include "netstrm.h" #include "errmsg.h" #include "tcpsrv.h" +#include "ruleset.h" #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT @@ -72,6 +73,7 @@ DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) DEFobjCurrIf(errmsg) +DEFobjCurrIf(ruleset) /* Module static data */ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ @@ -84,6 +86,7 @@ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mos static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ +static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ /* callbacks */ @@ -157,6 +160,27 @@ finalize_it: } +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imtcp current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; @@ -180,7 +204,8 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa } } - /* initialized, now add socket */ + /* initialized, now add socket and listener params */ + CHKiRet(tcpsrv.SetRuleset(pOurTcpsrv, pBindRuleset)); CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? UCHAR_CONSTANT("imtcp") : pszInputName)); tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); @@ -240,6 +265,7 @@ CODESTARTmodExit objRelease(tcps_sess, LM_TCPSRV_FILENAME); objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDmodExit @@ -249,14 +275,10 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus iTCPSessMax = 200; iStrmDrvrMode = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; - if(pszInputName != NULL) { - free(pszInputName); - pszInputName = NULL; - } - if(pszStrmDrvrAuthMode != NULL) { - free(pszStrmDrvrAuthMode); - pszStrmDrvrAuthMode = NULL; - } + free(pszInputName); + pszInputName = NULL; + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; return RS_RET_OK; } @@ -279,6 +301,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverrun"), 0, eCmdHdlrGetWord, @@ -295,6 +318,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverinputname"), 0, eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverbindruleset"), 0, + eCmdHdlrGetWord, setRuleset, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index e1e4a6a4..c39e54f6 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -814,7 +814,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy CHKiRet(cslcConstruct(&pThis, bChainingPermitted)); CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } /* important: add to list, AFTER everything else is OK. Else * we mess up things in the error case. @@ -825,7 +825,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(llAppend(&llCmdList, pMyCmdName, (void*) pThis)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } else { /* command already exists, are we allowed to chain? */ @@ -834,7 +834,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } diff --git a/runtime/conf.c b/runtime/conf.c index 81b6c081..dbc54fd4 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -1152,7 +1152,7 @@ cflineClassic(uchar *p, rule_t **ppRule) CHKiRet(ruleset.AddRule(rule.GetAssRuleset(*ppRule), ppRule)); } CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ - CHKiRet(rule.SetAssRuleset(*ppRule, pCurrRuleset)); /* create "fresh" selector */ + CHKiRet(rule.SetAssRuleset(*ppRule, ruleset.GetCurrent())); /* create "fresh" selector */ CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } diff --git a/runtime/msg.c b/runtime/msg.c index dbc3c779..10f283aa 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -46,6 +46,7 @@ #include "regexp.h" #include "atomic.h" #include "unicode-helper.h" +#include "ruleset.h" /* static data */ DEFobjStaticHelpers @@ -1166,13 +1167,21 @@ void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) } +/* rgerhards 2009-06-12: set associated ruleset + */ +void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) +{ + assert(pMsg != NULL); + pMsg->pRuleset = pRuleset; +} + + /* rgerhards 2004-11-16: set TAG in msg object */ void MsgSetTAG(msg_t *pMsg, char* pszTAG) { assert(pMsg != NULL); - if(pMsg->pszTAG != NULL) - free(pMsg->pszTAG); + free(pMsg->pszTAG); pMsg->iLenTAG = strlen(pszTAG); if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..b42f641f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -120,6 +120,7 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ }; @@ -167,6 +168,7 @@ char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); char *getTAG(msg_t *pM); int getHOSTNAMELen(msg_t *pM); diff --git a/runtime/obj.c b/runtime/obj.c index 8b9c9c83..f38b1d7f 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -1279,8 +1279,8 @@ objClassExit(void) /* TODO: implement the class exits! */ #if 0 - cfsyslineInit(pModInfo); - varClassInit(pModInfo); + cfsyslineExit(pModInfo); + varClassExit(pModInfo); #endif errmsgClassExit(); moduleClassExit(); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 3496bb0d..6f732f0e 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -152,12 +152,10 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wti"; - CHKiRet(wtiClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wtp"; - CHKiRet(wtpClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "queue"; - CHKiRet(qqueueClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok_token"; + CHKiRet(ctok_tokenClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok"; + CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmstk"; CHKiRet(vmstkClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "sysvar"; @@ -168,16 +166,18 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(vmopClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmprg"; CHKiRet(vmprgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok_token"; - CHKiRet(ctok_tokenClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok"; - CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "expr"; CHKiRet(exprClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "rule"; CHKiRet(ruleClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ruleset"; CHKiRet(rulesetClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wti"; + CHKiRet(wtiClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wtp"; + CHKiRet(wtpClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "queue"; + CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); diff --git a/runtime/ruleset.c b/runtime/ruleset.c index f9edde8b..93d40e24 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -40,6 +40,7 @@ #include "rsyslog.h" #include "obj.h" +#include "msg.h" #include "ruleset.h" #include "rule.h" #include "errmsg.h" @@ -53,6 +54,8 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(rule) linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ +ruleset_t *pCurrRuleset = NULL; /* currently "active" ruleset */ +ruleset_t *pDfltRuleset = NULL; /* currentl default ruleset, e.g. for binding to actions which have no other */ /* ---------- linked-list key handling functions ---------- */ @@ -147,12 +150,15 @@ DEFFUNC_llExecFunc(processMsgDoRules) * rgerhards, 2005-10-13 */ static rsRetVal -processMsg(ruleset_t *pThis, msg_t *pMsg) +processMsg(msg_t *pMsg) { + ruleset_t *pThis; DEFiRet; - ISOBJ_TYPE_assert(pThis, ruleset); assert(pMsg != NULL); + pThis = (pMsg->pRuleset == NULL) ? pDfltRuleset : pMsg->pRuleset; + ISOBJ_TYPE_assert(pThis, ruleset); + CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); finalize_it: @@ -200,6 +206,69 @@ finalize_it: } +/* get current ruleset + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static ruleset_t* +GetCurrent(void) +{ + return pCurrRuleset; +} + + +/* Find the ruleset with the given name and return a pointer to its object. + */ +static rsRetVal +GetRuleset(ruleset_t **ppRuleset, uchar *pszName) +{ + DEFiRet; + assert(ppRuleset != NULL); + assert(pszName != NULL); + + CHKiRet(llFind(&llRulesets, pszName, (void*) ppRuleset)); + +finalize_it: + RETiRet; +} + + +/* Set a new default rule set. If the default can not be found, no change happens. + */ +static rsRetVal +SetDefaultRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pDfltRuleset = pRuleset; + dbgprintf("default rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + +/* Set a new current rule set. If the ruleset can not be found, no change happens. + */ +static rsRetVal +SetCurrRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pCurrRuleset = pRuleset; + dbgprintf("current rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + /* destructor we need to destruct rules inside our linked list contents. */ static rsRetVal @@ -237,6 +306,13 @@ rulesetConstructFinalize(ruleset_t *pThis) CHKmalloc(keyName = ustrdup(pThis->pszName)); CHKiRet(llAppend(&llRulesets, keyName, pThis)); + /* this now also is the new current ruleset */ + pCurrRuleset = pThis; + + /* and also the default, if so far none has been set */ + if(pDfltRuleset == NULL) + pDfltRuleset = pThis; + finalize_it: RETiRet; } @@ -336,6 +412,10 @@ CODESTARTobjQueryInterface(ruleset) pIf->ProcessMsg = processMsg; pIf->SetName = setName; pIf->DebugPrintAll = debugPrintAll; + pIf->GetCurrent = GetCurrent; + pIf->GetRuleset = GetRuleset; + pIf->SetDefaultRuleset = SetDefaultRuleset; + pIf->SetCurrRuleset = SetCurrRuleset; finalize_it: ENDobjQueryInterface(ruleset) diff --git a/runtime/ruleset.h b/runtime/ruleset.h index b609e6b3..32571687 100644 --- a/runtime/ruleset.h +++ b/runtime/ruleset.h @@ -44,8 +44,12 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ rsRetVal (*IterateAllActions)(rsRetVal (*pFunc)(void*, void*), void* pParam); rsRetVal (*DestructAllActions)(void); rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); - rsRetVal (*ProcessMsg)(ruleset_t *pThis, msg_t *pMsg); rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); + rsRetVal (*ProcessMsg)(msg_t *pMsg); + rsRetVal (*GetRuleset)(ruleset_t **ppThis, uchar*); + rsRetVal (*SetDefaultRuleset)(uchar*); + rsRetVal (*SetCurrRuleset)(uchar*); + ruleset_t* (*GetCurrent)(void); ENDinterface(ruleset) #define rulesetCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tcps_sess.c b/tcps_sess.c index 62d51f66..d6bcd51b 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -254,6 +254,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pThis->fromHost); + MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); CHKiRet(submitMsg(pMsg)); diff --git a/tcpsrv.c b/tcpsrv.c index 3516b2e3..95409d2f 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -69,6 +69,7 @@ #include "netstrm.h" #include "nssel.h" #include "errmsg.h" +#include "ruleset.h" #include "unicode-helper.h" MODULE_TYPE_LIB @@ -81,6 +82,7 @@ MODULE_TYPE_LIB DEFobjStaticHelpers DEFobjCurrIf(conf) DEFobjCurrIf(glbl) +DEFobjCurrIf(ruleset) DEFobjCurrIf(tcps_sess) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) @@ -104,6 +106,8 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; +RUNLOG_VAR("%p", pThis->pRuleset); + pEntry->pRuleset = pThis->pRuleset; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); /* and add to list */ @@ -755,6 +759,16 @@ finalize_it: } +/* Set the ruleset (ptr) to use */ +static rsRetVal +SetRuleset(tcpsrv_t *pThis, ruleset_t *pRuleset) +{ + DEFiRet; + pThis->pRuleset = pRuleset; + RETiRet; +} + + /* here follows a number of methods that shuffle authentication settings down * to the drivers. Drivers not supporting these settings may return an error * state. @@ -855,6 +869,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetCBOnRegularClose = SetCBOnRegularClose; pIf->SetCBOnErrClose = SetCBOnErrClose; pIf->SetOnMsgReceive = SetOnMsgReceive; + pIf->SetRuleset = SetRuleset; finalize_it: ENDobjQueryInterface(tcpsrv) @@ -868,6 +883,7 @@ CODESTARTObjClassExit(tcpsrv) /* release objects we no longer need */ objRelease(tcps_sess, DONT_LOAD_LIB); objRelease(conf, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(netstrms, DONT_LOAD_LIB); @@ -891,6 +907,7 @@ BEGINObjClassInit(tcpsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(tcps_sess, DONT_LOAD_LIB)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, tcpsrvDebugPrint); diff --git a/tcpsrv.h b/tcpsrv.h index 2d174ce0..a6076e67 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -38,6 +38,7 @@ struct tcpLstnPortList_s { uchar *pszPort; /**< the ports the listener shall listen on */ uchar *pszInputName; /**< value to be used as input name */ tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ + ruleset_t *pRuleset; /**< associated ruleset */ tcpLstnPortList_t *pNext; /**< next port or NULL */ }; @@ -50,6 +51,7 @@ struct tcpsrv_s { int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ uchar *pszInputName; /**< value to be used as input name */ + ruleset_t *pRuleset; /**< ruleset to bind to */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ int iLstnMax; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ @@ -107,6 +109,7 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetSessMax)(tcpsrv_t *pThis, int iMaxSess); /* 2009-04-09 */ /* added v6 */ rsRetVal (*SetOnMsgReceive)(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); /* 2009-05-24 */ + rsRetVal (*SetRuleset)(tcpsrv_t *pThis, ruleset_t*); /* 2009-06-12 */ ENDinterface(tcpsrv) #define tcpsrvCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* change for v4: diff --git a/tools/syslogd.c b/tools/syslogd.c index 05c61059..99bf281d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -249,8 +249,6 @@ int repeatinterval[2] = { 30, 60 }; /* # of secs before flush */ #define LIST_DELIMITER ':' /* delimiter between two hosts */ -ruleset_t *pCurrRuleset; /* ruleset that is currently being processed */ - static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */ typedef struct legacyOptsLL_s { @@ -952,7 +950,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); } - ruleset.ProcessMsg(pCurrRuleset, pMsg); + ruleset.ProcessMsg(pMsg); msgDestruct(&pMsg); RETiRet; @@ -2172,6 +2170,7 @@ init(void) rsRetVal localRet; int iNbrActions; int bHadConfigErr = 0; + ruleset_t *pRuleset; char cbuf[BUFSIZ]; char bufStartUpMsg[512]; struct sigaction sigAct; @@ -2217,10 +2216,10 @@ init(void) conf.ReInitConf(); - // TODO: move to the right place - ruleset.Construct(&pCurrRuleset); - ruleset.SetName(pCurrRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); - ruleset.ConstructFinalize(pCurrRuleset); + /* construct the default ruleset */ + ruleset.Construct(&pRuleset); + ruleset.SetName(pRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); + ruleset.ConstructFinalize(pRuleset); /* open the configuration file */ localRet = conf.processConfFile(ConfFile); @@ -2259,7 +2258,7 @@ init(void) } else { dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } - ruleset.AddRule(pCurrRuleset, &pRule); + ruleset.AddRule(ruleset.GetCurrent(), &pRule); } legacyOptsHook(); @@ -2398,25 +2397,45 @@ finalize_it: } -/* Begin a new rule set. The new rule set is created, and all rules that now - * follow go into that rule set. - * TODO: we may later add the capability to switch back to an already existing - * rule set. - * NOTE: pCurrRuleset is NOT desructed and must not be! The ruleset class keeps - * a list of all known rule sets, and can destruct them at the end of execution. - * pCurrRuleset is just a shortcut so that "everyone" knows which ruleset to - * extend. - * TODO: A problem with this function is the way config lines are processed. The rule - * is actually only written when the next rule is completely read. That way, this - * (past) rule goes into the wrong (new) ruleset. I need to see how to fix this best... - * rgerhards, 2009-06-10 +/* Switch the default ruleset (that, what servcies bind to if nothing specific + * is specified). + * rgerhards, 2009-06-12 + */ +static rsRetVal +setDefaultRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + DEFiRet; + + CHKiRet(ruleset.SetDefaultRuleset(pszName)); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + +/* Switch to either an already existing rule set or start a new one. The + * named rule set becomes the new "current" rule set (what means that new + * actions are added to it). + * rgerhards, 2009-06-12 */ -static rsRetVal beginNewRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +static rsRetVal +setCurrRuleset(void __attribute__((unused)) *pVal, uchar *pszName) { + ruleset_t *pRuleset; + rsRetVal localRet; DEFiRet; - CHKiRet(ruleset.Construct(&pCurrRuleset)); - CHKiRet(ruleset.SetName(pCurrRuleset, pszName)); - CHKiRet(ruleset.ConstructFinalize(pCurrRuleset)); + + localRet = ruleset.SetCurrRuleset(pszName); + + if(localRet == RS_RET_NOT_FOUND) { + DBGPRINTF("begin new current rule set '%s'\n", pszName); + CHKiRet(ruleset.Construct(&pRuleset)); + CHKiRet(ruleset.SetName(pRuleset, pszName)); + CHKiRet(ruleset.ConstructFinalize(pRuleset)); + } else { + ABORT_FINALIZE(localRet); + } finalize_it: free(pszName); /* no longer needed */ @@ -2658,7 +2677,7 @@ static rsRetVal loadBuildInModules(void) } /* dirty, but this must be for the time being: the usrmsg module must always be - * loaded as last module. This is because it processes any time of action selector. + * loaded as last module. This is because it processes any type of action selector. * If we load it before other modules, these others will never have a chance of * working with the config file. We may change that implementation so that a user name * must start with an alnum, that would definitely help (but would it break backwards @@ -2666,8 +2685,7 @@ static rsRetVal loadBuildInModules(void) * User names now must begin with: * [a-zA-Z0-9_.] */ - if((iRet = module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)) != RS_RET_OK) - RETiRet; + CHKiRet(module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)); /* ok, initialization of the command handler probably does not 100% belong right in * this space here. However, with the current design, this is actually quite a good @@ -2677,7 +2695,8 @@ static rsRetVal loadBuildInModules(void) * This, I think, is the right thing to do. -- rgerhards, 2007-07-31 */ CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeretrycount", 0, eCmdHdlrInt, NULL, &glbliActionResumeRetryCount, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, beginNewRuleset, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"defaultruleset", 0, eCmdHdlrGetWord, setDefaultRuleset, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, setCurrRuleset, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszMainMsgQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesize", 0, eCmdHdlrInt, NULL, &iMainMsgQueueSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iMainMsgQHighWtrMark, NULL)); -- cgit From b5ccdf06724d309b777d4dd38a455ed2ef0318a0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 15:59:50 +0200 Subject: performance-enhanced imtcp ...by now using lowres time and thus saving many time() calls. This needs some performance testing and must be made configurable if it works out. --- plugins/imudp/imudp.c | 31 ++++++++++++++++++++++++++++++- tcps_sess.c | 12 ++++++++++-- tcpsrv.c | 1 - 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index a486b818..50e8efaf 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -68,6 +68,7 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * it so that we can check available memory in willRun() and request * termination if we can not get it. -- rgerhards, 2007-12-27 */ +// TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -97,7 +98,7 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) else bindAddr = pszBindAddr; - dbgprintf("Trying to open syslog UDP ports at %s:%s.\n", + DBGPRINTF("Trying to open syslog UDP ports at %s:%s.\n", (bindAddr == NULL) ? (uchar*)"*" : bindAddr, pNewVal); newSocks = net.create_udp_socket(bindAddr, (pNewVal == NULL || *pNewVal == '\0') ? (uchar*) "514" : pNewVal, 1); @@ -137,6 +138,30 @@ finalize_it: } +#if 0 /* TODO: implement when tehre is time, requires restructure of socket array! */ +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal +setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imudp current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} +#endif + + /* This function is a helper to runInput. I have extracted it * from the main loop just so that we do not have that large amount of code * in a single place. This function takes a socket and pulls messages from @@ -386,6 +411,10 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ + /* TODO: add - but this requires more changes, no time right now... + CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserverbindruleset", 0, eCmdHdlrGetWord, + setRuleset, NULL, STD_LOADABLE_MODULE_ID)); + */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserverrun", 0, eCmdHdlrGetWord, addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, diff --git a/tcps_sess.c b/tcps_sess.c index d6bcd51b..8ba5cbec 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -56,6 +56,11 @@ DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ +#define TIME_REQUERY_DFLT 16 // TODO change back! 2 +static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ +static int iNbrTimeUsed = 0; /* how often has previous time been used so far? */ + + /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); @@ -240,9 +245,10 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } - //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { +RUNLOG_STR("XXX: quering time!"); datetime.getCurrTime(&stTime, &ttGenTime); - //} + } /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ @@ -307,6 +313,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); + iNbrTimeUsed = 0; /* full time query */ defaultDoSubmitMessage(pThis); } @@ -451,6 +458,7 @@ DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ + iNbrTimeUsed = 0; /* full time query */ while(pData < pEnd) { CHKiRet(processDataRcvd(pThis, *pData++)); } diff --git a/tcpsrv.c b/tcpsrv.c index 95409d2f..5e2bd530 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -106,7 +106,6 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; -RUNLOG_VAR("%p", pThis->pRuleset); pEntry->pRuleset = pThis->pRuleset; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); -- cgit From 057c9b11fca013205877e15dd851927a11aa058b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 16:30:39 +0200 Subject: fixed a regression from past commit & more performance enhancement a larger buffer is a cheap, yet useful, enhancement... ;) --- tcps_sess.c | 5 +++-- tcpsrv.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tcps_sess.c b/tcps_sess.c index 8ba5cbec..f01114d6 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -234,8 +234,8 @@ static rsRetVal defaultDoSubmitMessage(tcps_sess_t *pThis) { msg_t *pMsg; - struct syslogTime stTime; - time_t ttGenTime; + static struct syslogTime stTime; /* the static vars are currently OK (single input thread!) */ + static time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -245,6 +245,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } +RUNLOG_VAR("%ld", ttGenTime); if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { RUNLOG_STR("XXX: quering time!"); datetime.getCurrTime(&stTime, &ttGenTime); diff --git a/tcpsrv.c b/tcpsrv.c index 5e2bd530..21699bca 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -513,7 +513,7 @@ Run(tcpsrv_t *pThis) while(nfds && iTCPSess != -1) { CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + char buf[64*1024]; /* reception buffer - may hold a partial or multiple messages */ dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); /* Receive message */ -- cgit From 1f874c220860d3a19fb6cfb60f0902a08639f6ab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 17:46:16 +0200 Subject: fixed small bug in testbench --- tcps_sess.c | 2 -- tests/diskqueue-fsync.sh | 1 + tests/diskqueue.sh | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tcps_sess.c b/tcps_sess.c index f01114d6..1446df10 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -245,9 +245,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } -RUNLOG_VAR("%ld", ttGenTime); if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { -RUNLOG_STR("XXX: quering time!"); datetime.getCurrTime(&stTime, &ttGenTime); } /* we now create our own message object and submit it to the queue */ diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh index 45fb864c..0282202d 100755 --- a/tests/diskqueue-fsync.sh +++ b/tests/diskqueue-fsync.sh @@ -11,5 +11,6 @@ source $srcdir/diag.sh startup diskqueue-fsync.conf # 1000 messages should be enough - the disk fsync test is very slow! source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 1000 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown source $srcdir/diag.sh seq-check 0 999 source $srcdir/diag.sh exit diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 2fe31db9..668a1224 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -11,5 +11,6 @@ source $srcdir/diag.sh startup diskqueue.conf # 20000 messages should be enough - the disk test is slow enough ;) source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown source $srcdir/diag.sh seq-check 0 19999 source $srcdir/diag.sh exit -- cgit From 16ecb90c3ac88bb2261c31c990d88f97f1a1b32f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Jun 2009 13:44:51 +0200 Subject: omfile buffers are now synchronized after inactivity This is the first shot at this functionality. Currently, we run off a fixed counter in the rsyslogd mainloop, which needs to be restructured. But this code works, so it is a good time for a commit. --- runtime/Makefile.am | 2 + runtime/apc.c | 400 ++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/apc.h | 56 ++++++++ runtime/obj.c | 2 + runtime/srUtils.h | 13 ++ runtime/stream.c | 68 +++++++++ runtime/stream.h | 10 +- tools/omfile.c | 7 + tools/syslogd.c | 44 +----- 9 files changed, 559 insertions(+), 43 deletions(-) create mode 100644 runtime/apc.c create mode 100644 runtime/apc.h diff --git a/runtime/Makefile.am b/runtime/Makefile.am index de0daac4..c2ef7cfa 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -39,6 +39,8 @@ librsyslog_la_SOURCES = \ obj.h \ modules.c \ modules.h \ + apc.c \ + apc.h \ sync.c \ sync.h \ expr.c \ diff --git a/runtime/apc.c b/runtime/apc.c new file mode 100644 index 00000000..b0b5f298 --- /dev/null +++ b/runtime/apc.c @@ -0,0 +1,400 @@ +/* apc.c - asynchronous procedure call support + * + * An asynchronous procedure call (APC) is a procedure call (guess what) that is potentially run + * asynchronously to its main thread. It can be scheduled to occur at a caller-provided time. + * As long as the procedure has not been called, the APC entry may be modified by the caller + * or deleted. It is the caller's purpose to make sure proper synchronization is in place. + * The APC object only case about APC's own control structures (which *are* properly + * guarded by synchronization primitives). + * + * Module begun 2009-06-15 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "apc.h" +#include "srUtils.h" + +/* static data */ +DEFobjStaticHelpers + +/* following is a used to implement a monotonically increasing id for the apcs. That + * ID can be used to cancel an apc request. Note that the ID is generated with modulo + * arithmetic, so at some point, it will wrap. Howerver, this happens at 2^32-1 at + * earliest, so this is not considered a problem. + */ +apc_id_t apcID = 0; + +/* private data structures */ + +/* the apc list and its entries + * This is a doubly-linked list as we need to be able to do inserts + * and deletes right in the middle of the list. It is inspired by the + * Unix callout mechanism. + * Note that we support two generic caller-provided parameters as + * experience shows that at most two are often used. This causes very + * little overhead, but simplifies caller code in cases where exactly + * two parameters are needed. We hope this is a useful optimizaton. + * rgerhards, 2009-06-15 + */ +typedef struct apc_list_s { + struct apc_list_s *pNext; + struct apc_list_s *pPrev; + apc_id_t id; + apc_t *pApc; /* pointer to the APC object to be scheduled */ +} apc_list_t; + +apc_list_t *apcListRoot = NULL; +apc_list_t *apcListTail = NULL; +pthread_mutex_t listMutex; /* needs to be locked for all list operations */ + + +/* destructor for the apc object */ +BEGINobjDestruct(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(apc) +ENDobjDestruct(apc) + + +/* ------------------------------ APC list handling functions ------------------------------ */ + +/* Function that handles changes to the list root. Most importantly, this function + * needs to schedule a new timer. It is OK to call this function with an empty list. + */ +static rsRetVal +listRootChanged(void) +{ + DEFiRet; + + if(apcListRoot == NULL) + FINALIZE; + + // TODO: implement! + +finalize_it: + RETiRet; +} + + +/* insert an apc entry into the APC list. The same entry MUST NOT already be present! + */ +static rsRetVal +insertApc(apc_t *pThis, apc_id_t *pID) +{ + apc_list_t *pCurr; + apc_list_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (apc_list_t*) calloc(1, sizeof(apc_list_t))); + pNew->pApc = pThis; + pNew->id = *pID = apcID++; +dbgprintf("insert apc %p, id %ld\n", pThis, pNew->id); + + /* find right list location */ + if(apcListRoot == NULL) { + /* no need to search, list is empty */ + apcListRoot = pNew; + apcListTail = pNew; + CHKiRet(listRootChanged()); + } else { + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->pApc->ttExec > pThis->ttExec) + break; + } + + if(pCurr == NULL) { + /* insert at tail */ + pNew->pPrev = apcListTail; + apcListTail->pNext = pNew; + apcListTail = pNew; + } else { + if(pCurr == apcListRoot) { + /* new first entry */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + apcListRoot = pNew; + CHKiRet(listRootChanged()); + } else { + /* in the middle of the list */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + } + } + } + + +finalize_it: + RETiRet; +} + + +/* Delete an apc entry from the APC list. It is OK if the entry is not found, + * in this case we assume it already has been processed. + */ +static rsRetVal +deleteApc(apc_id_t id) +{ + apc_list_t *pCurr; + DEFiRet; + +dbgprintf("trying to delete apc %ld\n", id); + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->id == id) { +RUNLOG_STR("apc id found, now deleting!\n"); + if(pCurr == apcListRoot) { + apcListRoot = pCurr->pNext; + CHKiRet(listRootChanged()); + } else { + pCurr->pPrev->pNext = pCurr->pNext; + } + if(pCurr->pNext == NULL) { + apcListTail = pCurr->pPrev; + } else { + pCurr->pNext->pPrev = pCurr->pPrev; + } + free(pCurr); + pCurr = NULL; + break; + } + } + +finalize_it: + RETiRet; +} + + +/* unlist all elements up to the current timestamp. Return this as a seperate list + * to the caller. Returns an empty (NULL ptr) list if there are no such elements. + * The caller must handle that gracefully. The list is returned in the parameter. + */ +static rsRetVal +unlistCurrent(apc_list_t **ppList) +{ + apc_list_t *pCurr; + time_t tCurr; + DEFiRet; + assert(ppList != NULL); + + time(&tCurr); + + if(apcListRoot == NULL || apcListRoot->pApc->ttExec > tCurr) { + *ppList = NULL; + FINALIZE; + } + + *ppList = apcListRoot; + /* now search up to which entry we need to execute */ + for(pCurr = apcListRoot ; pCurr != NULL && pCurr->pApc->ttExec <= tCurr ; pCurr = pCurr->pNext) { + /*JUST SKIP TO LAST ELEMENT*/; + } + + if(pCurr == NULL) { + /* all elements can be unlisted */ + apcListRoot = NULL; + apcListTail = NULL; + } else { + /* need to set a new root */ + pCurr->pPrev->pNext = NULL; /* terminate newly unlisted list */ + pCurr->pPrev = NULL; /* we are the new root */ + apcListRoot = pCurr; + } + +finalize_it: + RETiRet; +} + + +/* ------------------------------ END APC list handling functions ------------------------------ */ + + +/* execute all list elements that are currently scheduled for execution. We do this in two phases. + * In the first phase, we look the list mutex and move everything from the head of the queue to + * the current timestamp to a new to-be-executed list. Then we unlock the mutex and do the actual + * exec (which may take some time). + * Note that the caller is responsible for proper + * caller-level synchronization. The caller may schedule another Apc, this module must + * ensure that (and it does so by not locking the list mutex while we call the Apc). + * Note: this function "consumes" the apc_t, so it is no longer existing after this + * function returns. + */ +// TODO make static and associated with our own pthread-based timer +rsRetVal +execScheduled(void) +{ + apc_list_t *pExecList; + apc_list_t *pCurr; + apc_list_t *pNext; + DEFVARS_mutexProtection_uncond; + DEFiRet; + + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + iRet = unlistCurrent(&pExecList); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + CHKiRet(iRet); + + DBGPRINTF("running apc scheduler - we have %s to execute\n", + pExecList == NULL ? "nothing" : "something"); + for(pCurr = pExecList ; pCurr != NULL ; pCurr = pNext) { +dbgprintf("executing apc list entry %p, apc %p\n", pCurr, pCurr->pApc); + pNext = pCurr->pNext; + pCurr->pApc->pProc(pCurr->pApc->param1, pCurr->pApc->param2); + apcDestruct(&pCurr->pApc); + free(pCurr); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(apc) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(apc) + + +/* ConstructionFinalizer + * Note that we use a non-standard calling interface: pID returns the current APC + * id. This is the only way to handle the situation without the need for extra + * locking. + * rgerhards, 2008-01-09 + */ +static rsRetVal +apcConstructFinalize(apc_t *pThis, apc_id_t *pID) +{ + DEFVARS_mutexProtection_uncond; + DEFiRet; + ISOBJ_TYPE_assert(pThis, apc); + assert(pID != NULL); + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + insertApc(pThis, pID); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); +RUNLOG_STR("apcConstructFinalize post mutex unlock\n"); + RETiRet; +} + + +/* some set methods */ +static rsRetVal +SetProcedure(apc_t *pThis, void (*pProc)(void*, void*)) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->pProc = pProc; + return RS_RET_OK; +} +static rsRetVal +SetParam1(apc_t *pThis, void *param1) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param1; + return RS_RET_OK; +} +static rsRetVal +SetParam2(apc_t *pThis, void *param2) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param2; + return RS_RET_OK; +} + + +/* cancel an Apc request, ID is provided. It is OK if the ID can not be found, this may + * happen if the Apc was executed in the mean time. So it is safe to call CancelApc() at + * any time. + */ +static rsRetVal +CancelApc(apc_id_t id) +{ + DEFVARS_mutexProtection_uncond; + + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + deleteApc(id); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + return RS_RET_OK; +} + + +/* debugprint for the apc object */ +BEGINobjDebugPrint(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(apc) + dbgoprint((obj_t*) pThis, "APC module, currently no state info available\n"); +ENDobjDebugPrint(apc) + + +/* queryInterface function + */ +BEGINobjQueryInterface(apc) +CODESTARTobjQueryInterface(apc) + if(pIf->ifVersion != apcCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = apcConstruct; + pIf->ConstructFinalize = apcConstructFinalize; + pIf->Destruct = apcDestruct; + pIf->DebugPrint = apcDebugPrint; + pIf->CancelApc = CancelApc; + pIf->SetProcedure = SetProcedure; + pIf->SetParam1 = SetParam1; + pIf->SetParam2 = SetParam2; +finalize_it: +ENDobjQueryInterface(apc) + + +/* Exit the apc class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(apc, OBJ_IS_CORE_MODULE) /* class, version */ + //objRelease(apcstk, CORE_COMPONENT); + pthread_mutex_destroy(&listMutex); +ENDObjClassExit(apc) + + +/* Initialize the apc class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(apc, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + //CHKiRet(objUse(apcstk, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, apcDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, apcConstructFinalize); + + /* do other initializations */ + pthread_mutex_init(&listMutex, NULL); +ENDObjClassInit(apc) + +/* vi:set ai: + */ diff --git a/runtime/apc.h b/runtime/apc.h new file mode 100644 index 00000000..7c679b97 --- /dev/null +++ b/runtime/apc.h @@ -0,0 +1,56 @@ +/* The apc object. + * + * See apc.c for more information. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_APC_H +#define INCLUDED_APC_H + +/* the apc object */ +typedef struct apc_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + time_t ttExec; /* when to call procedure (so far seconds...) */ + void (*pProc)(void*, void*); /* which procedure to call */ + void *param1; /* user-supplied parameters */ + void *param2; /* user-supplied parameters */ +} apc_t; + +typedef unsigned long apc_id_t; /* monotonically incrementing apc ID */ + +/* interfaces */ +BEGINinterface(apc) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(apc); + rsRetVal (*Construct)(apc_t **ppThis); + rsRetVal (*ConstructFinalize)(apc_t *pThis, apc_id_t *); + rsRetVal (*Destruct)(apc_t **ppThis); + rsRetVal (*SetProcedure)(apc_t *pThis, void (*pProc)(void*, void*)); + rsRetVal (*SetParam1)(apc_t *pThis, void *); + rsRetVal (*SetParam2)(apc_t *pThis, void *); + rsRetVal (*CancelApc)(apc_id_t); +ENDinterface(apc) +#define apcCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(apc); + +#endif /* #ifndef INCLUDED_APC_H */ diff --git a/runtime/obj.c b/runtime/obj.c index f38b1d7f..41991853 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -88,6 +88,7 @@ #include "errmsg.h" #include "cfsysline.h" #include "unicode-helper.h" +#include "apc.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ @@ -1313,6 +1314,7 @@ objClassInit(modInfo_t *pModInfo) CHKiRet(objGetObjInterface(&obj)); /* get ourselves ;) */ /* init classes we use (limit to as few as possible!) */ + CHKiRet(apcClassInit(pModInfo)); CHKiRet(errmsgClassInit(pModInfo)); CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); diff --git a/runtime/srUtils.h b/runtime/srUtils.h index 699f8527..b37559cf 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -125,4 +125,17 @@ rsRetVal getFileSize(uchar *pszName, off_t *pSize); d_pthread_mutex_unlock(mut); \ pthread_setcancelstate(iCancelStateSave, NULL); \ } + +/* The unconditional versions of the macro always lock the mutex. They are preferred in + * complex scenarios, where the simple ones might get mixed up by multiple calls. + */ +#define DEFVARS_mutexProtection_uncond\ + int iCancelStateSave +#define BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ + d_pthread_mutex_lock(mut); +#define END_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ + d_pthread_mutex_unlock(mut); \ + pthread_setcancelstate(iCancelStateSave, NULL); + #endif diff --git a/runtime/stream.c b/runtime/stream.c index c8672aa2..773da319 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -39,6 +39,7 @@ #include #include /* required for HP UX */ #include +#include #include "rsyslog.h" #include "stringbuf.h" @@ -47,10 +48,12 @@ #include "stream.h" #include "unicode-helper.h" #include "module-template.h" +#include "apc.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(zlibw) +DEFobjCurrIf(apc) /* forward definitions */ static rsRetVal strmFlush(strm_t *pThis); @@ -60,6 +63,20 @@ static rsRetVal strmCloseFile(strm_t *pThis); /* methods */ +/* async flush apc handler + */ +static void +flushApc(void *param1, void __attribute__((unused)) *param2) +{ + DEFVARS_mutexProtection_uncond; + strm_t *pThis = (strm_t*) param1; + ISOBJ_TYPE_assert(pThis, strm); + + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); + strmFlush(pThis); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); +} + /* Try to resolve a size limit situation. This is used to support custom-file size handlers * for omfile. It first runs the command, and then checks if we are still above the size @@ -583,6 +600,11 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } } + /* if we should call flush apc's, we need a mutex */ + if(pThis->iFlushInterval != 0) { + pthread_mutex_init(&pThis->mut, 0); + } + finalize_it: RETiRet; } @@ -602,6 +624,11 @@ CODESTARTobjDestruct(strm) objRelease(zlibw, LM_ZLIBW_FILENAME); } + if(pThis->iFlushInterval != 0) { + // TODO: check if there is an apc and remove it! + pthread_mutex_destroy(&pThis->mut); + } + free(pThis->pszDir); free(pThis->pIOBuf); free(pThis->pZipBuf); @@ -965,11 +992,33 @@ finalize_it: } +/* schedule an Apc flush request. + * rgerhards, 2009-06-15 + */ +static inline rsRetVal +scheduleFlushRequest(strm_t *pThis) +{ + apc_t *pApc; + DEFiRet; + + CHKiRet(apc.CancelApc(pThis->apcID)); +dbgprintf("XXX: requesting to add apc!\n"); + CHKiRet(apc.Construct(&pApc)); + CHKiRet(apc.SetProcedure(pApc, (void (*)(void*, void*))flushApc)); + CHKiRet(apc.SetParam1(pApc, pThis)); + CHKiRet(apc.ConstructFinalize(pApc, &pThis->apcID)); + +finalize_it: + RETiRet; +} + + /* write memory buffer to a stream object */ static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { + DEFVARS_mutexProtection_uncond; DEFiRet; size_t iPartial; @@ -980,6 +1029,11 @@ dbgprintf("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n if(pThis->bDisabled) ABORT_FINALIZE(RS_RET_STREAM_DISABLED); +RUNLOG_VAR("%d", pThis->iFlushInterval); + if(pThis->iFlushInterval != 0) { + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); + } + /* check if the to-be-written data is larger than our buffer size */ if(lenBuf >= pThis->sIOBufSize) { /* it is - so we do a direct write, that is most efficient. @@ -1008,7 +1062,17 @@ dbgprintf("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n } } + /* we ignore the outcome of scheduleFlushRequest(), as we will write the data always at + * termination. For Zip mode, it could be fatal if we write after each record. + */ + if(pThis->iFlushInterval != 0) + scheduleFlushRequest(pThis); + finalize_it: + if(pThis->iFlushInterval != 0) { + END_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); + } + RETiRet; } @@ -1025,6 +1089,7 @@ DEFpropSetMeth(strm, iZipLevel, int) DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) DEFpropSetMeth(strm, iSizeLimit, off_t) +DEFpropSetMeth(strm, iFlushInterval, int) DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) @@ -1308,6 +1373,7 @@ CODESTARTobjQueryInterface(strm) pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; pIf->SetiSizeLimit = strmSetiSizeLimit; + pIf->SetiFlushInterval = strmSetiFlushInterval; pIf->SetpszSizeLimitCmd = strmSetpszSizeLimitCmd; finalize_it: ENDobjQueryInterface(strm) @@ -1319,6 +1385,8 @@ ENDobjQueryInterface(strm) */ BEGINObjClassInit(strm, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ + CHKiRet(objUse(apc, CORE_COMPONENT)); + OBJSetMethodHandler(objMethod_SERIALIZE, strmSerialize); OBJSetMethodHandler(objMethod_SETPROPERTY, strmSetProperty); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmConstructFinalize); diff --git a/runtime/stream.h b/runtime/stream.h index 021c4792..e3ad32b1 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -70,6 +70,7 @@ #include "glbl.h" #include "stream.h" #include "zlibw.h" +#include "apc.h" /* stream types */ typedef enum { @@ -118,6 +119,10 @@ typedef struct strm_s { bool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; + /* support for async flush procesing */ + int iFlushInterval; /* flush in which interval - 0, no flushing */ + apc_id_t apcID; /* id of current Apc request (used for cancelling) */ + pthread_mutex_t mut;/* mutex for flush in async mode */ /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ @@ -127,7 +132,7 @@ typedef struct strm_s { /* interfaces */ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Construct)(strm_t **ppThis); - rsRetVal (*ConstructFinalize)(strm_t __attribute__((unused)) *pThis); + rsRetVal (*ConstructFinalize)(strm_t *pThis); rsRetVal (*Destruct)(strm_t **ppThis); rsRetVal (*SetMaxFileSize)(strm_t *pThis, int64 iMaxFileSize); rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); @@ -157,9 +162,10 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, bSync, int); INTERFACEpropSetMeth(strm, sIOBufSize, size_t); INTERFACEpropSetMeth(strm, iSizeLimit, off_t); + INTERFACEpropSetMeth(strm, iFlushInterval, int); INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); ENDinterface(strm) -#define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/tools/omfile.c b/tools/omfile.c index 6d9eb096..675d313e 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -88,6 +88,7 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ +#define FLUSH_INTRVL_DFLT 1 /* default buffer flush interval (in seconds) */ /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -103,6 +104,7 @@ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathn static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ +static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -132,6 +134,7 @@ typedef struct _instanceData { uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ + int iFlushInterval; /* how fast flush buffer on inactivity? */ bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; @@ -400,6 +403,7 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); + CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); @@ -678,6 +682,7 @@ CODESTARTparseSelectorAct pData->iZipLevel = iZipLevel; pData->bFlushOnTXEnd = bFlushOnTXEnd; pData->iIOBufSize = iIOBufSize; + pData->iFlushInterval = iFlushInterval; if(pData->bDynamicName == 0) { /* try open and emit error message if not possible. At this stage, we ignore the @@ -712,6 +717,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iZipLevel = 0; bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; + iFlushInterval = FLUSH_INTRVL_DFLT; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -760,6 +766,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); diff --git a/tools/syslogd.c b/tools/syslogd.c index 99bf281d..2b29cd7c 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2575,7 +2575,8 @@ mainloop(void) * powertop, for example). In that case, we primarily wait for a signal, * but a once-a-day wakeup should be quite acceptable. -- rgerhards, 2008-06-09 */ - tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/; + //tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/; +tvSelectTimeout.tv_sec = 5; // TESTING ONLY!!! TODO: change back!!! tvSelectTimeout.tv_usec = 0; select(1, NULL, NULL, NULL, &tvSelectTimeout); if(bFinished) @@ -2610,49 +2611,11 @@ mainloop(void) bHadHUP = 0; continue; } + execScheduled(); /* handle Apc calls (if any) */ } ENDfunc } -/* If user is not root, prints warnings or even exits - * TODO: check all dynafiles for write permission - * ... but it is probably better to wait here until we have - * a module interface - rgerhards, 2007-07-23 - */ -static void checkPermissions() -{ -#if 0 - /* TODO: this function must either be redone or removed - now with the input modules, - * there is no such simple check we can do. What we can check, however, is if there is - * any input module active and terminate, if not. -- rgerhards, 2007-12-26 - */ - /* we are not root */ - if (geteuid() != 0) - { - fputs("WARNING: Local messages will not be logged! If you want to log them, run rsyslog as root.\n",stderr); -#ifdef SYSLOG_INET - /* udp enabled and port number less than or equal to 1024 */ - if ( AcceptRemote && (atoi(LogPort) <= 1024) ) - fprintf(stderr, "WARNING: Will not listen on UDP port %s. Use port number higher than 1024 or run rsyslog as root!\n", LogPort); - - /* tcp enabled and port number less or equal to 1024 */ - if( bEnableTCP && (atoi(TCPLstnPort) <= 1024) ) - fprintf(stderr, "WARNING: Will not listen on TCP port %s. Use port number higher than 1024 or run rsyslog as root!\n", TCPLstnPort); - - /* Neither explicit high UDP port nor explicit high TCP port. - * It is useless to run anymore */ - if( !(AcceptRemote && (atoi(LogPort) > 1024)) && !( bEnableTCP && (atoi(TCPLstnPort) > 1024)) ) - { -#endif - fprintf(stderr, "ERROR: Nothing to log, no reason to run. Please run rsyslog as root.\n"); - exit(EXIT_FAILURE); -#ifdef SYSLOG_INET - } -#endif - } -#endif -} - /* load build-in modules * very first version begun on 2007-07-23 by rgerhards @@ -3053,7 +3016,6 @@ doGlblProcessInit(void) int i; DEFiRet; - checkPermissions(); thrdInit(); if( !(Debug || NoFork) ) -- cgit From 4af40abc0a9cdc9f328830b6074f6182f43c171a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 07:15:48 +0200 Subject: cleaned up small nit --- runtime/unicode-helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 8216e992..7a776f68 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -40,7 +40,7 @@ #else static inline uchar* ustrncpy(uchar *psz1, uchar *psz2, size_t len) { - return strncpy((char*) psz1, (char*) psz2, len); + return (uchar*) strncpy((char*) psz1, (char*) psz2, len); } static inline uchar* ustrdup(uchar *psz) -- cgit From 015d17ca70e81ad998e32cdfeed3cd925fd7dedc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 08:46:45 +0200 Subject: some performance optimizations - saved gettimeofday() calls in imtcp (and increased reception buffer) - somewhat optimized stringbuf.c - some other optimizations --- plugins/imdiag/imdiag.c | 2 +- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- plugins/imudp/imudp.c | 4 ++-- runtime/msg.c | 57 ++++++++++++++++++++----------------------------- runtime/msg.h | 48 ++++++++++++++++++----------------------- runtime/obj-types.h | 4 ++-- runtime/parser.c | 17 +++++++++++++++ runtime/rsyslog.h | 2 ++ runtime/stringbuf.c | 6 ++---- tcps_sess.c | 54 ++++++++++++++++++++++------------------------ tcpsrv.c | 3 ++- tcpsrv.h | 1 + tools/syslogd.c | 4 ++-- 14 files changed, 102 insertions(+), 104 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index c700cab7..51f319ca 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -207,7 +207,7 @@ doInjectMsg(int iNum) CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg)); pMsg->iLenRawMsg = ustrlen(szMsg); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 92fd30c3..e3882ce5 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -95,7 +95,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index ecb6c100..45933436 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -95,7 +95,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsg(pMsg, (char*)msg); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index a486b818..57aec9b6 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -220,7 +220,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); pMsg->iLenRawMsg = lenRcvBuf; - MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp"), sizeof("imudp")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; diff --git a/runtime/msg.c b/runtime/msg.c index dbc3c779..b3c76089 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -104,6 +104,7 @@ static syslogCODE rs_facilitynames[] = /* some forward declarations */ static int getAPPNAMELen(msg_t *pM); +static int getProtocolVersion(msg_t *pM); /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we @@ -364,7 +365,6 @@ CODESTARTobjDestruct(msg) free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); - free(pThis->pszPRI); if(pThis->pCSProgName != NULL) rsCStrDestruct(&pThis->pCSProgName); if(pThis->pCSStrucData != NULL) @@ -670,7 +670,7 @@ void setProtocolVersion(msg_t *pM, int iNewVersion) pM->iProtocolVersion = iNewVersion; } -int getProtocolVersion(msg_t *pM) +static int getProtocolVersion(msg_t *pM) { assert(pM != NULL); return(pM->iProtocolVersion); @@ -689,7 +689,7 @@ int getMSGLen(msg_t *pM) } -char *getRawMsg(msg_t *pM) +static char *getRawMsg(msg_t *pM) { if(pM == NULL) return ""; @@ -724,7 +724,7 @@ char *getMSG(msg_t *pM) /* Get PRI value in text form */ -char *getPRI(msg_t *pM) +static char *getPRI(msg_t *pM) { int pri; BEGINfunc @@ -742,7 +742,7 @@ char *getPRI(msg_t *pM) * report. -- rgerhards, 2008-07-14 */ pri = pM->iFacility * 8 + pM->iSeverity; - if((pM->pszPRI = malloc(5)) == NULL) return ""; + pM->pszPRI = pM->bufPRI; pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); } MsgUnlock(pM); @@ -753,14 +753,14 @@ char *getPRI(msg_t *pM) /* Get PRI value as integer */ -int getPRIi(msg_t *pM) +static int getPRIi(msg_t *pM) { assert(pM != NULL); return (pM->iFacility << 3) + (pM->iSeverity); } -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -838,7 +838,7 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) return "INVALID eFmt OPTION!"; } -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -917,7 +917,7 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) } -char *getSeverity(msg_t *pM) +static inline char *getSeverity(msg_t *pM) { if(pM == NULL) return ""; @@ -934,7 +934,7 @@ char *getSeverity(msg_t *pM) } -char *getSeverityStr(msg_t *pM) +static inline char *getSeverityStr(msg_t *pM) { syslogCODE *c; int val; @@ -964,7 +964,7 @@ char *getSeverityStr(msg_t *pM) return((char*)pM->pszSeverityStr); } -char *getFacility(msg_t *pM) +static inline char *getFacility(msg_t *pM) { if(pM == NULL) return ""; @@ -983,7 +983,7 @@ char *getFacility(msg_t *pM) return((char*)pM->pszFacility); } -char *getFacilityStr(msg_t *pM) +static inline char *getFacilityStr(msg_t *pM) { syslogCODE *c; int val; @@ -1090,7 +1090,7 @@ finalize_it: /* rgerhards, 2005-11-24 */ -int getPROCIDLen(msg_t *pM) +static inline int getPROCIDLen(msg_t *pM) { assert(pM != NULL); MsgLock(pM); @@ -1135,19 +1135,10 @@ finalize_it: RETiRet; } -/* rgerhards, 2005-11-24 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getMSGIDLen(msg_t *pM) -{ - return (pM->pCSMSGID == NULL) ? 1 : rsCStrLen(pM->pCSMSGID); -} -#endif - /* rgerhards, 2005-11-24 */ -char *getMSGID(msg_t *pM) +static inline char *getMSGID(msg_t *pM) { return (pM->pCSMSGID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSMSGID); } @@ -1171,8 +1162,7 @@ void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) void MsgSetTAG(msg_t *pMsg, char* pszTAG) { assert(pMsg != NULL); - if(pMsg->pszTAG != NULL) - free(pMsg->pszTAG); + free(pMsg->pszTAG); pMsg->iLenTAG = strlen(pszTAG); if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); @@ -1229,7 +1219,7 @@ static int getTAGLen(msg_t *pM) #endif -char *getTAG(msg_t *pM) +static inline char *getTAG(msg_t *pM) { char *ret; @@ -1272,7 +1262,7 @@ char *getHOSTNAME(msg_t *pM) } -uchar *getInputName(msg_t *pM) +static uchar *getInputName(msg_t *pM) { if(pM == NULL) return (uchar*) ""; @@ -1339,7 +1329,7 @@ static int getStructuredDataLen(msg_t *pM) /* get the "STRUCTURED-DATA" as sz string * rgerhards, 2005-11-24 */ -char *getStructuredData(msg_t *pM) +static inline char *getStructuredData(msg_t *pM) { return (pM->pCSStrucData == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSStrucData); } @@ -1465,14 +1455,13 @@ static int getAPPNAMELen(msg_t *pM) } /* rgerhards 2008-09-10: set pszInputName in msg object + * rgerhards, 2009-06-16 */ -void MsgSetInputName(msg_t *pMsg, uchar* pszInputName) +void MsgSetInputName(msg_t *pMsg, uchar* pszInputName, size_t lenInputName) { assert(pMsg != NULL); - if(pMsg->pszInputName != NULL) - free(pMsg->pszInputName); - - pMsg->iLenInputName = ustrlen(pszInputName); + free(pMsg->pszInputName); + pMsg->iLenInputName = lenInputName; if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); } @@ -2493,7 +2482,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..1689bbbc 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -120,6 +120,9 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ + /* now follow fixed-size buffers to safe some time otherwise used for allocs */ + uchar bufPRI[5]; + }; @@ -137,45 +140,20 @@ short bDoLock; /* use the mutex? */ /* function prototypes */ PROTOTYPEObjClassInit(msg); -char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -int getProtocolVersion(msg_t *pM); -char *getProtocolVersionString(msg_t *pM); -int getMSGLen(msg_t *pM); -char *getRawMsg(msg_t *pM); -char *getUxTradMsg(msg_t *pM); -char *getMSG(msg_t *pM); -char *getPRI(msg_t *pM); -int getPRIi(msg_t *pM); -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt); -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt); -char *getSeverity(msg_t *pM); -char *getSeverityStr(msg_t *pM); -char *getFacility(msg_t *pM); -char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, uchar*); +void MsgSetInputName(msg_t *pMsg, uchar*, size_t); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); -char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); -int getPROCIDLen(msg_t *pM); -char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, char* pszTAG); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); -char *getTAG(msg_t *pM); -int getHOSTNAMELen(msg_t *pM); -char *getHOSTNAME(msg_t *pM); -uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -char *getStructuredData(msg_t *pM); -int getProgramNameLen(msg_t *pM); -char *getProgramName(msg_t *pM); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); @@ -184,13 +162,29 @@ int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); -char *getMSGID(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +/* TODO: remove these five (so far used in action.c) */ +char *getMSG(msg_t *pM); +char *getHOSTNAME(msg_t *pM); +char *getPROCID(msg_t *pM); +char *getAPPNAME(msg_t *pM); +int getMSGLen(msg_t *pM); + +char *getHOSTNAME(msg_t *pM); +int getHOSTNAMELen(msg_t *pM); +char *getProgramName(msg_t *pM); +int getProgramNameLen(msg_t *pM); +uchar *getRcvFrom(msg_t *pM); + +#if 0 +char *getUxTradMsg(msg_t *pM); +#endif + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains * us quite some performance and is fully abstracted using this header file. diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 914c2f2c..78829f94 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -280,7 +280,7 @@ rsRetVal objName##ClassExit(void) \ * rgerhards, 2008-01-30 */ #define BEGINobjDestruct(OBJ) \ - rsRetVal OBJ##Destruct(OBJ##_t **ppThis) \ + rsRetVal OBJ##Destruct(OBJ##_t __attribute__((unused)) **ppThis) \ { \ DEFiRet; \ int iCancelStateSave; \ @@ -314,7 +314,7 @@ rsRetVal objName##ClassExit(void) \ #define PROTOTYPEObjDebugPrint(obj) rsRetVal obj##DebugPrint(obj##_t *pThis) #define INTERFACEObjDebugPrint(obj) rsRetVal (*DebugPrint)(obj##_t *pThis) #define BEGINobjDebugPrint(obj) \ - rsRetVal obj##DebugPrint(obj##_t *pThis) \ + rsRetVal obj##DebugPrint(obj##_t __attribute__((unused)) *pThis) \ { \ DEFiRet; \ diff --git a/runtime/parser.c b/runtime/parser.c index 212d40f3..64e03094 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -191,6 +191,23 @@ sanitizeMessage(msg_t *pMsg) lenMsg--; } + /* it is much quicker to sweep over the message and see if it actually + * needs sanitation than to do the sanitation in any case. So we first do + * this and terminate when it is not needed - which is expectedly the case + * for the vast majority of messages. -- rgerhards, 2009-06-15 + */ + int bNeedSanitize = 0; + for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { + if(pszMsg[iSrc] < 32) { + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + bNeedSanitize = 1; + break; + } + } + } + if(bNeedSanitize == 0) + FINALIZE; + /* now copy over the message and sanitize it */ /* TODO: can we get cheaper memory alloc? {alloca()?}*/ iMaxLine = glbl.GetMaxLine(); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ea303a51..32177a9f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -103,6 +103,8 @@ typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif +typedef char bool; /* I intentionally use char, to keep it slim so that many fit into the CPU cache! */ + /* settings for flow control * TODO: is there a better place for them? -- rgerhards, 2008-03-14 */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 07256fab..f3d9aa48 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -299,10 +299,8 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->pBuf != NULL) - free(pThis->pBuf); - if(pThis->pszBuf != NULL) - free(pThis->pszBuf); + free(pThis->pBuf); + free(pThis->pszBuf); if(pszNew == NULL) { pThis->iStrLen = 0; pThis->iBufSize = 0; diff --git a/tcps_sess.c b/tcps_sess.c index 62d51f66..cfee0523 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -97,12 +97,9 @@ CODESTARTobjDestruct(tcps_sess) pThis->pSrv->pOnSessDestruct(&pThis->pUsr); } /* now destruct our own properties */ - if(pThis->fromHost != NULL) - free(pThis->fromHost); - if(pThis->fromHostIP != NULL) - free(pThis->fromHostIP); - if(pThis->pMsg != NULL) - free(pThis->pMsg); + free(pThis->fromHost); + free(pThis->fromHostIP); + free(pThis->pMsg); ENDobjDestruct(tcps_sess) @@ -124,10 +121,7 @@ SetHost(tcps_sess_t *pThis, uchar *pszHost) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHost != NULL) { - free(pThis->fromHost); - } - + free(pThis->fromHost); pThis->fromHost = pszHost; RETiRet; @@ -144,10 +138,7 @@ SetHostIP(tcps_sess_t *pThis, uchar *pszHostIP) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHostIP != NULL) { - free(pThis->fromHostIP); - } - + free(pThis->fromHostIP); pThis->fromHostIP = pszHostIP; RETiRet; @@ -226,11 +217,9 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis) +defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime) { msg_t *pMsg; - struct syslogTime stTime; - time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -240,16 +229,13 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } - //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); - //} /* we now create our own message object and submit it to the queue */ - CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); pMsg->iLenRawMsg = pThis->iMsg; - MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); + MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName, pThis->pLstnInfo->lenInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; @@ -281,6 +267,8 @@ finalize_it: static rsRetVal PrepareClose(tcps_sess_t *pThis) { + struct syslogTime stTime; + time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -306,7 +294,8 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - defaultDoSubmitMessage(pThis); + datetime.getCurrTime(&stTime, &ttGenTime); + defaultDoSubmitMessage(pThis, &stTime, ttGenTime); } finalize_it: @@ -341,7 +330,7 @@ Close(tcps_sess_t *pThis) * rgerhards, 2008-03-14 */ static rsRetVal -processDataRcvd(tcps_sess_t *pThis, char c) +processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -387,7 +376,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -398,7 +387,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -415,7 +404,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); pThis->inputState = eAtStrtFram; } } @@ -436,22 +425,29 @@ processDataRcvd(tcps_sess_t *pThis, char c) * RS_RET_OK, which means the session should be kept open * or anything else, which means it must be closed. * rgerhards, 2008-03-01 + * As a performance optimization, we pick up the timestamp here. Acutally, + * this *is* the *correct* reception step for all the data we received, because + * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { - DEFiRet; + struct syslogTime stTime; + time_t ttGenTime; char *pEnd; + DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); assert(pData != NULL); assert(iLen > 0); + datetime.getCurrTime(&stTime, &ttGenTime); + /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++)); + CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime)); } finalize_it: diff --git a/tcpsrv.c b/tcpsrv.c index 3516b2e3..11619498 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -105,6 +105,7 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) pEntry->pszPort = pszPort; pEntry->pSrv = pThis; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + pEntry->lenInputName = ustrlen(pEntry->pszInputName); /* and add to list */ pEntry->pNext = pThis->pLstnPorts; @@ -510,7 +511,7 @@ Run(tcpsrv_t *pThis) while(nfds && iTCPSess != -1) { CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); /* Receive message */ diff --git a/tcpsrv.h b/tcpsrv.h index 2d174ce0..95537d54 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -37,6 +37,7 @@ typedef enum ETCPsyslogFramingAnomaly { struct tcpLstnPortList_s { uchar *pszPort; /**< the ports the listener shall listen on */ uchar *pszInputName; /**< value to be used as input name */ + size_t lenInputName; /**< length of inputName */ tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ tcpLstnPortList_t *pNext; /**< next port or NULL */ }; diff --git a/tools/syslogd.c b/tools/syslogd.c index b43b7a37..a9f1dbdb 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -639,7 +639,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) - MsgSetInputName(pMsg, pszInputName); + MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName)); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsg(pMsg, (char*)msg); @@ -947,7 +947,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); -- cgit From f7579e68a67364c8040966be57c2eae4c9550ee5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 11:36:05 +0200 Subject: done various optimizations to the stringbuf and its users --- outchannel.c | 20 ++--- parse.c | 16 ++-- runtime/cfsysline.c | 12 +-- runtime/conf.c | 40 +++++----- runtime/ctok.c | 4 +- runtime/datetime.c | 5 +- runtime/msg.c | 30 ++----- runtime/msg.h | 4 +- runtime/nsd_gtls.c | 8 +- runtime/obj.c | 6 +- runtime/parser.c | 19 ++++- runtime/stream.c | 2 +- runtime/stringbuf.c | 221 ++++++++++++++++++---------------------------------- runtime/stringbuf.h | 27 +++---- runtime/vm.c | 2 +- runtime/vmop.c | 2 +- template.c | 54 ++++++------- tools/syslogd.c | 170 ++++++++++++++++++++-------------------- 18 files changed, 282 insertions(+), 360 deletions(-) diff --git a/outchannel.c b/outchannel.c index 5c348b63..4f8abb32 100644 --- a/outchannel.c +++ b/outchannel.c @@ -105,22 +105,22 @@ static rsRetVal get_Field(uchar **pp, uchar **pField) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p && *p != ' ' && *p != ',') { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pField, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pField, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -174,22 +174,22 @@ static inline rsRetVal get_restOfLine(uchar **pp, uchar **pBuf) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p) { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pBuf, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pBuf, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; diff --git a/parse.c b/parse.c index 58458d62..87c67676 100644 --- a/parse.c +++ b/parse.c @@ -268,7 +268,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim /* We got the string, now take it and see if we need to * remove anything at its end. */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); if(bTrimTrailing) { CHKiRet(rsCStrTrimTrailingWhiteSpace(pCStr)); @@ -344,7 +344,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) } /* We got the string, let's finish it... */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* done! */ *ppCStr = pCStr; @@ -380,7 +380,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) assert(pIP != NULL); assert(pBits != NULL); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); parsSkipWhitespace(pThis); pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; @@ -390,8 +390,8 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) */ while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != '/' && *pC != ',' && !isspace((int)*pC)) { - if((iRet = rsCStrAppendChar(pCStr, *pC)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrAppendChar(pCStr, *pC)) != RS_RET_OK) { + cstrDestruct (&pCStr); FINALIZE; } ++pThis->iCurrPos; @@ -399,15 +399,15 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) } /* We got the string, let's finish it... */ - if((iRet = rsCStrFinish(pCStr)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrFinalize(pCStr)) != RS_RET_OK) { + cstrDestruct(&pCStr); FINALIZE; } /* now we have the string and must check/convert it to * an NetAddr structure. */ - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pszIP, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pszIP, 0)); *pIP = calloc(1, sizeof(struct NetAddr)); diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index e1e4a6a4..0fb4247d 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -462,7 +462,7 @@ getWord(uchar **pp, cstr_t **ppStrB) ASSERT(*pp != NULL); ASSERT(ppStrB != NULL); - CHKiRet(rsCStrConstruct(ppStrB)); + CHKiRet(cstrConstruct(ppStrB)); skipWhiteSpace(pp); /* skip over any whitespace */ @@ -470,9 +470,9 @@ getWord(uchar **pp, cstr_t **ppStrB) p = *pp; while(*p && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(*ppStrB, *p++)); + CHKiRet(cstrAppendChar(*ppStrB, *p++)); } - CHKiRet(rsCStrFinish(*ppStrB)); + CHKiRet(cstrFinalize(*ppStrB)); *pp = p; @@ -506,7 +506,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); pStrB = NULL; /* we got the word, now set it */ @@ -525,7 +525,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -548,7 +548,7 @@ doSyslogName(uchar **pp, rsRetVal (*pSetHdlr)(void*, int), void *pVal, syslogNam ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); /* get word */ - iNewVal = decodeSyslogName(rsCStrGetSzStr(pStrB), pNameTable); + iNewVal = decodeSyslogName(cstrGetSzStr(pStrB), pNameTable); if(pSetHdlr == NULL) { /* we should set value directly to var */ diff --git a/runtime/conf.c b/runtime/conf.c index 7cdcf5ec..65fc66f1 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -526,17 +526,15 @@ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEn tplName = (uchar*) strdup((char*)dfltTplName); } else { /* template specified, pick it up */ - if(rsCStrConstruct(&pStrB) != RS_RET_OK) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKiRet(cstrConstruct(&pStrB)); /* now copy the string */ while(*p && *p != '#' && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &tplName, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &tplName, 0)); } iRet = OMSRsetEntry(pOMSR, iEntry, tplName, iTplOpts); @@ -925,7 +923,7 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) */ static rsRetVal cflineProcessHostSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -951,21 +949,20 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) dbgprintf("resetting BSD-like hostname filter\n"); eDfltHostnameCmpMode = HN_NO_COMP; if(pDfltHostnameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, NULL)); } } else { dbgprintf("setting BSD-like hostname filter to '%s'\n", *pline); if(pDfltHostnameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } @@ -976,7 +973,7 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) */ static rsRetVal cflineProcessTagSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -995,21 +992,20 @@ static rsRetVal cflineProcessTagSelector(uchar **pline) if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { dbgprintf("resetting programname filter\n"); if(pDfltProgNameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, NULL)); } } else { dbgprintf("setting programname filter to '%s'\n", *pline); if(pDfltProgNameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } diff --git a/runtime/ctok.c b/runtime/ctok.c index d2cd8bbd..263e656c 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -269,7 +269,7 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ - CHKiRet(rsCStrFinish(pstrVal)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -319,7 +319,7 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(ctokGetCharFromStream(pThis, &c)); } - CHKiRet(rsCStrFinish(pStrB)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; diff --git a/runtime/datetime.c b/runtime/datetime.c index 19e61a0a..40ab4e9c 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -152,11 +152,10 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) static int srSLMGParseInt32(uchar** ppsz) { - int i; + register int i; i = 0; - while(isdigit((int) **ppsz)) - { + while(isdigit((int) **ppsz)) { i = i * 10 + **ppsz - '0'; ++(*ppsz); } diff --git a/runtime/msg.c b/runtime/msg.c index b3c76089..55cc48b8 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -437,11 +437,12 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); + pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); tmpCOPYSZ(SeverityStr); tmpCOPYSZ(Facility); tmpCOPYSZ(FacilityStr); - tmpCOPYSZ(PRI); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); tmpCOPYSZ(UxTradMsg); @@ -587,7 +588,7 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) } /* OK, finaally we could obtain a PROCID. So let's use it ;) */ - CHKiRet(rsCStrFinish(pM->pCSPROCID)); + CHKiRet(cstrFinalize(pM->pCSPROCID)); finalize_it: RETiRet; @@ -629,7 +630,7 @@ static rsRetVal aquireProgramName(msg_t *pM) ; ++i) { CHKiRet(rsCStrAppendChar(pM->pCSProgName, pM->pszTAG[i])); } - CHKiRet(rsCStrFinish(pM->pCSProgName)); + CHKiRet(cstrFinalize(pM->pCSProgName)); } finalize_it: RETiRet; @@ -724,31 +725,12 @@ char *getMSG(msg_t *pM) /* Get PRI value in text form */ -static char *getPRI(msg_t *pM) +static inline char *getPRI(msg_t *pM) { - int pri; - BEGINfunc - if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszPRI == NULL) { - /* OK, we need to construct it... we use a 5 byte buffer - as of - * RFC 3164, it can't be longer. Should it still be, snprintf will truncate... - * Note that we do not use the LOG_MAKEPRI macro. This macro - * is a simple add of the two values under FreeBSD 7. So we implement - * the logic in our own code. This is a change from a bug - * report. -- rgerhards, 2008-07-14 - */ - pri = pM->iFacility * 8 + pM->iSeverity; - pM->pszPRI = pM->bufPRI; - pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); - } - MsgUnlock(pM); - - ENDfunc - return (char*)pM->pszPRI; + return (char*)pM->bufPRI; } diff --git a/runtime/msg.h b/runtime/msg.h index 1689bbbc..20360641 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -73,7 +73,8 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - uchar *pszPRI; /* the PRI as a string */ + //uchar *pszPRI; /* the PRI as a string */ + uchar bufPRI[5]; int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we @@ -121,7 +122,6 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ /* now follow fixed-size buffers to safe some time otherwise used for allocs */ - uchar bufPRI[5]; }; diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 1a50e2f8..19dc8678 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -335,7 +335,7 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) gnutls_x509_crt_deinit(cert); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; finalize_it: @@ -455,7 +455,7 @@ GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) snprintf((char*)buf, sizeof(buf), ":%2.2X", pFingerprint[i]); CHKiRet(rsCStrAppendStrWithLen(pStr, buf, 3)); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -723,7 +723,7 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) } ++i; /* char processed */ } - CHKiRet(rsCStrFinish(pstrCN)); + CHKiRet(cstrFinalize(pstrCN)); /* we got it - we ignore the rest of the DN string (if any). So we may * not detect if it contains more than one CN @@ -884,7 +884,7 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) if(!bFoundPositiveMatch) { dbgprintf("invalid peer name, not permitted to talk to it\n"); if(pThis->bReportAuthErr == 1) { - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer name not authorized - " "not permitted to talk to it. Names: %s", diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..355c0f97 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -447,7 +447,7 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -515,7 +515,7 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pCStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* check terminator */ if(c != ':') ABORT_FINALIZE(RS_RET_INVALID_DELIMITER); @@ -629,7 +629,7 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pProp->pcsName, c)); NEXTC; } - CHKiRet(rsCStrFinish(pProp->pcsName)); + CHKiRet(cstrFinalize(pProp->pcsName)); /* property type */ CHKiRet(objDeserializeNumber(&i, pStrm)); diff --git a/runtime/parser.c b/runtime/parser.c index 64e03094..13fb51ec 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -205,8 +205,16 @@ sanitizeMessage(msg_t *pMsg) } } } - if(bNeedSanitize == 0) + if(bNeedSanitize == 0) { + /* what a shame - we do not have a \0 byte... + * TODO: think about adding it or otherwise be able to use it... + */ + uchar *pRaw; + CHKmalloc(pRaw = realloc(pMsg->pszRawMsg, pMsg->iLenRawMsg + 1)); + pRaw[pMsg->iLenRawMsg] = '\0'; + pMsg->pszRawMsg = pRaw; FINALIZE; + } /* now copy over the message and sanitize it */ /* TODO: can we get cheaper memory alloc? {alloca()?}*/ @@ -276,6 +284,7 @@ rsRetVal parseMsg(msg_t *pMsg) DEFiRet; uchar *msg; int pri; + int iPriText; CHKiRet(sanitizeMessage(pMsg)); @@ -285,11 +294,19 @@ rsRetVal parseMsg(msg_t *pMsg) /* pull PRI */ pri = DEFUPRI; msg = pMsg->pszRawMsg; + iPriText = 0; if(*msg == '<') { + /* while we process the PRI, we also fill the PRI textual representation + * inside the msg object. This may not be ideal from an OOP point of view, + * but it offers us performance... + */ pri = 0; while(isdigit((int) *++msg)) { + pMsg->bufPRI[iPriText++ % 4] = *msg; /* mod 4 to guard against malformed messages! */ pri = 10 * pri + (*msg - '0'); } + pMsg->bufPRI[iPriText % 4] = '\0'; + pMsg->iLenPRI = iPriText % 4; if(*msg == '>') ++msg; if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..1cff2da6 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -368,7 +368,7 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr) CHKiRet(rsCStrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } - CHKiRet(rsCStrFinish(*ppCStr)); + CHKiRet(cstrFinalize(*ppCStr)); finalize_it: if(iRet != RS_RET_OK && *ppCStr != NULL) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index f3d9aa48..a2d9c599 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -6,8 +6,9 @@ * Please see syslogd.c for license information. * All functions in this "class" start with rsCStr (rsyslog Counted String). * begun 2005-09-07 rgerhards + * did some optimization (read: bugs!) rgerhards, 2009-06-16 * - * Copyright (C) 2007-2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2007-2009 by Rainer Gerhards and Adiscon GmbH * * This file is part of the rsyslog runtime library. * @@ -40,6 +41,7 @@ #include "regexp.h" #include "obj.h" +uchar* rsCStrGetSzStr(cstr_t *pThis); /* ################################################################# * * private members * @@ -54,15 +56,14 @@ DEFobjCurrIf(regexp) * ################################################################# */ -rsRetVal rsCStrConstruct(cstr_t **ppThis) +rsRetVal cstrConstruct(cstr_t **ppThis) { DEFiRet; cstr_t *pThis; ASSERT(ppThis != NULL); - if((pThis = (cstr_t*) calloc(1, sizeof(cstr_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pThis = (cstr_t*) calloc(1, sizeof(cstr_t))); rsSETOBJTYPE(pThis, OIDrsCStr); pThis->pBuf = NULL; @@ -89,7 +90,7 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); - pThis->iBufSize = pThis->iStrLen = strlen((char*)(char *) sz); + pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -137,24 +138,8 @@ void rsCStrDestruct(cstr_t **ppThis) { cstr_t *pThis = *ppThis; - /* rgerhards 2005-10-19: The free of pBuf was contained in conditional compilation. - * The code was only compiled if STRINGBUF_TRIM_ALLOCSIZE was set to 1. I honestly - * do not know why it was so, I think it was an artifact. Anyhow, I have changed this - * now. Should there any issue occur, this comment hopefully will shed some light - * on what happened. I re-verified, and this function has never before been called - * by anyone. So changing it can have no impact for obvious reasons... - * - * rgerhards, 2008-02-20: I changed the interface to the new calling conventions, where - * the destructor receives a pointer to the object, so that it can set it to NULL. - */ - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } - - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - } - + free(pThis->pBuf); + free(pThis->pszBuf); RSFREEOBJ(pThis); *ppThis = NULL; } @@ -166,12 +151,14 @@ void rsCStrDestruct(cstr_t **ppThis) * allocated. In practice, a bit more is allocated because we envision that * some more characters may be added after these. * rgerhards, 2008-01-07 + * changed to utilized realloc() -- rgerhards, 2009-06-16 */ -static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) +static rsRetVal +rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { - DEFiRet; uchar *pNewBuf; size_t iNewSize; + DEFiRet; /* first compute the new size needed */ if(iMinNeeded > pThis->iAllocIncrement) { @@ -187,15 +174,9 @@ static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) } iNewSize += pThis->iBufSize; /* add current size */ - /* and then allocate and copy over */ /* DEV debugging only: dbgprintf("extending string buffer, old %d, new %d\n", pThis->iBufSize, iNewSize); */ - if((pNewBuf = (uchar*) malloc(iNewSize * sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - memcpy(pNewBuf, pThis->pBuf, pThis->iBufSize); + CHKmalloc(pNewBuf = (uchar*) realloc(pThis->pBuf, iNewSize * sizeof(uchar))); pThis->iBufSize = iNewSize; - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } pThis->pBuf = pNewBuf; finalize_it: @@ -288,6 +269,29 @@ finalize_it: } +/* NEW VARIANT + * Append a character to the current string object. This may only be done until + * cstrFinalize() is called. + * rgerhards, 2009-06-16 + */ +rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) +{ + DEFiRet; + + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + if(pThis->iStrLen >= pThis->iBufSize) { + CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ + } + + /* ok, when we reach this, we have sufficient memory */ + *(pThis->pBuf + pThis->iStrLen++) = c; + +finalize_it: + RETiRet; +} + + /* Sets the string object to the classigal sz-string provided. * Any previously stored vlaue is discarded. If a NULL pointer * the the new value (pszNew) is provided, an empty string is @@ -388,35 +392,42 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) } +/* NEW VERSION for interface without separate psz buffer! */ +/* Returns the cstr data as a classical C sz string. We use that the + * Finalizer did properly terminate our string (but we may stil be NULL). + * So it is vital that the finalizer is called BEFORe this function here! + * The caller must not free or otherwise manipulate the returned string and must not + * destroy the CStr object as long as the ascii string is used. + * This function may return NULL, if the string is currently NULL. This + * is a feature, not a bug. If you need non-NULL in any case, use + * cstrGetSzStrNoNULL() instead. + * Note that due to the new single-buffer interface this function almost does nothing! + * rgerhards, 2006-09-16 + */ +uchar* cstrGetSzStr(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + return(pThis->pBuf); +} + + /* Converts the CStr object to a classical zero-terminated C string, * returns that string and destroys the CStr object. The returned string * MUST be freed by the caller. The function might return NULL if * no memory can be allocated. * - * TODO: - * This function should at some time become special. The base idea is to - * add one extra byte to the end of the regular buffer, so that we can - * convert it to an szString without the need to copy. The extra memory - * footprint is not hefty, but the performance gain is potentially large. - * To get it done now, I am not doing the optimiziation right now. - * rgerhards, 2005-09-07 + * This is the NEW replacement for rsCStrConvSzStrAndDestruct which does + * no longer utilize a special buffer but soley works on pBuf (and also + * assumes that cstrFinalize had been called). * - * rgerhards, 2007-09-04: I have changed the interface of this function. It now - * returns an rsRetVal, so that we can communicate back if we have an error. - * Using the standard method is much better than returning NULL. Secondly, NULL - * was not actually an error - it was in indication if the string was empty. - * This was needed in some parts of the code, in others not. I have now added - * a second parameter to specify what the caller needs. I hope these changes - * will make it less likely that the function is called incorrectly, what - * previously happend quite often and was the cause of a number of program - * aborts. So the parameters are now: + * Parameters are as follows: * pointer to the object, pointer to string-pointer to receive string and * bRetNULL: 0 - must not return NULL on empty string, return "" in that * case, 1 - return NULL instead of an empty string. * PLEASE NOTE: the caller must free the memory returned in ppSz in any case * (except, of course, if it is NULL). */ -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) { DEFiRet; uchar* pRetBuf; @@ -427,14 +438,13 @@ rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - if((pRetBuf = malloc(sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pRetBuf = malloc(sizeof(uchar))); *pRetBuf = '\0'; } else { pRetBuf = NULL; } } else - pRetBuf = rsCStrGetSzStr(pThis); + pRetBuf = pThis->pBuf; *ppSz = pRetBuf; @@ -443,60 +453,39 @@ finalize_it: * that we can NOT use the rsCStrDestruct function as it would * also free the sz String buffer, which we pass on to the user. */ - if(pThis->pBuf != NULL) - free(pThis->pBuf); RSFREEOBJ(pThis); - RETiRet; } -#if STRINGBUF_TRIM_ALLOCSIZE == 1 - /* Only in this mode, we need to trim the string. To do - * so, we must allocate a new buffer of the exact - * string size, and then copy the old one over. - */ - /* WARNING - * STRINGBUF_TRIM_ALLOCSIZE can, in theory, be used to trim - * memory buffers. This part of the code was inherited from - * liblogging (where it is used in a different context) but - * never put to use in rsyslog. The reason is that it is hardly - * imaginable where the extra performance cost is worth the save - * in memory alloc. Then Anders Blomdel rightfully pointed out that - * the code does not work at all - and nobody even know that it - * probably shouldn't. Rather than removing, I deciced to somewhat - * fix the code, so that this feature may be enabled if somebody - * really has a need for it. Be warned, however, that I NEVER - * tested the fix. So if you intend to use this feature, you must - * do full testing before you rely on it. -- rgerhards, 2008-02-12 - */ -rsRetVal rsCStrFinish(cstr_t __attribute__((unused)) *pThis) +/* Finalize the string object. This must be called after all data is added to it + * but before that data is used. + * rgerhards, 2009-06-16 + */ +rsRetVal +cstrFinalize(cstr_t *pThis) { DEFiRet; - uchar* pBuf; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + assert(pThis->bIsFinalized == 0); - if((pBuf = malloc((pThis->iStrLen) * sizeof(uchar))) == NULL) - { /* OK, in this case we use the previous buffer. At least - * we have it ;) - */ - } - else - { /* got the new buffer, so let's use it */ - memcpy(pBuf, pThis->pBuf, pThis->iStrLen); - pThis->pBuf = pBuf; + if(pThis->iStrLen > 0) { + /* terminate string only if one exists */ + CHKiRet(cstrAppendChar(pThis, '\0')); + --pThis->iStrLen; /* do NOT count the \0 byte */ } + pThis->bIsFinalized = 1; +finalize_it: RETiRet; } -#endif /* #if STRINGBUF_TRIM_ALLOCSIZE == 1 */ void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); assert(iNewIncrement > 0); - pThis->iAllocIncrement = iNewIncrement; } @@ -1025,56 +1014,6 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz) } -#if 0 /* read comment below why this is commented out. In short: for future use! */ -/* locate the first occurence of a standard sz string inside a rsCStr object. - * Returns the offset (0-bound) of this first occurrence. If not found, -1 is - * returned. - * rgerhards 2005-09-19 - * WARNING: I accidently created this function (I later noticed I didn't relly - * need it... I will not remove the function, as it probably is useful - * some time later. However, it is not fully tested, so start with testing - * it before you put it to first use). - */ -int rsCStrLocateSzStr(cstr_t *pThis, uchar *sz) -{ - int iLenSz; - int i; - int iMax; - int bFound; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(sz == NULL) - return 0; - - iLenSz = strlen((char*)sz); - if(iLenSz == 0) - return 0; - - /* compute the largest index where a match could occur - after all, - * the to-be-located string must be able to be present in the - * searched string (it needs its size ;)). - */ - iMax = pThis->iStrLen - iLenSz; - - bFound = 0; - i = 0; - while(i < iMax && !bFound) { - int iCheck; - uchar *pComp = pThis->pBuf + i; - for(iCheck = 0 ; iCheck < iLenSz ; ++iCheck) - if(*(pComp + iCheck) != *(sz + iCheck)) - break; - if(iCheck == iLenSz) - bFound = 1; /* found! - else it wouldn't be equal */ - else - ++i; /* on to the next try */ - } - - return(bFound ? i : -1); -} -#endif /* end comment out */ - - /* our exit function. TODO: remove once converted to a class * rgerhards, 2008-03-11 */ @@ -1098,11 +1037,5 @@ finalize_it: } -/* - * Local variables: - * c-indent-level: 8 - * c-basic-offset: 8 - * tab-width: 8 - * End: - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 684133bb..d28aee26 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -49,13 +49,15 @@ typedef struct cstr_s size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ size_t iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ + bool bIsFinalized; /**< is this object finished and ready for use? (a debug aid, may be removed later TODO 2009-06-16) */ } cstr_t; /** * Construct a rsCStr object. */ -rsRetVal rsCStrConstruct(cstr_t **ppThis); +rsRetVal cstrConstruct(cstr_t **ppThis); +#define rsCStrConstruct(x) cstrConstruct((x)) rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz); rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); @@ -63,6 +65,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); * Destruct the string buffer object. */ void rsCStrDestruct(cstr_t **ppThis); +#define cstrDestruct(x) rsCStrDestruct((x)) /** * Append a character to an existing string. If necessary, the @@ -71,6 +74,7 @@ void rsCStrDestruct(cstr_t **ppThis); * \param c Character to append to string. */ rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); +rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); /** * Truncate "n" number of characters from the end of the @@ -123,10 +127,9 @@ rsRetVal rsCStrAppendInt(cstr_t *pThis, long i); rsRetVal strExit(void); /* TODO: remove once we have a real object interface! */ -uchar* rsCStrGetSzStr(cstr_t *pThis); +uchar* __attribute__((deprecated)) rsCStrGetSzStr(cstr_t *pThis); uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis); rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew); -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); int rsCStrCStrCmp(cstr_t *pCS1, cstr_t *pCS2); int rsCStrSzStrCmp(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrOffsetSzStrCmp(cstr_t *pCS1, size_t iOffset, uchar *psz, size_t iLenSz); @@ -142,6 +145,11 @@ rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); +/* new calling interface */ +rsRetVal cstrFinalize(cstr_t *pThis); +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); +uchar* cstrGetSzStr(cstr_t *pThis); + /* now come inline-like functions */ #ifdef NDEBUG # define rsCStrLen(x) ((int)((x)->iStrLen)) @@ -149,19 +157,6 @@ rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); int rsCStrLen(cstr_t *pThis); #endif -#if STRINGBUF_TRIM_ALLOCSIZE != 1 -/* This is the normal case (see comment in rsCStrFinish!). In those cases, the function - * simply needs to do nothing, so that we can save us the function call. - * rgerhards, 2008-02-12 - */ -# define rsCStrFinish(pThis) RS_RET_OK -#else - /** - * Finish the string buffer dynamic allocation. - */ - rsRetVal rsCStrFinish(cstr_t *pThis); -#endif - #define rsCStrGetBufBeg(x) ((x)->pBuf) rsRetVal strInit(); diff --git a/runtime/vm.c b/runtime/vm.c index 125b0d21..8cbf9e12 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -563,7 +563,7 @@ rsf_tolower(vmstk_t *pStk, int numOperands) } /* Store result and cleanup */ - CHKiRet(rsCStrFinish(pcstr)); + CHKiRet(cstrFinalize(pcstr)); var.SetString(operand1, pcstr); vmstk.Push(pStk, operand1); finalize_it: diff --git a/runtime/vmop.c b/runtime/vmop.c index 3e001d27..acacfc9e 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -85,7 +85,7 @@ CODESTARTobjDebugPrint(vmop) CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); } } - CHKiRet(rsCStrFinish(&pStrVar)); + CHKiRet(cstrFinalize(pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); if(pThis->opcode != opcode_FUNC_CALL) rsCStrDestruct(&pStrVar); diff --git a/template.c b/template.c index 4a12aa3b..6e34bcd8 100644 --- a/template.c +++ b/template.c @@ -88,7 +88,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) * free the obtained value (if requested). We continue this * loop until we got hold of all values. */ - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); pTpe = pTpl->pEntryRoot; while(pTpe != NULL) { @@ -99,7 +99,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) ) { dbgprintf("error %d during tplToString()\n", iRet); /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); FINALIZE; } } else if(pTpe->eEntryType == FIELD) { @@ -119,7 +119,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, (uchar*) pVal, iLenVal)) { dbgprintf("error %d during tplToString()\n", iRet); /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); if(bMustBeFreed) free(pVal); FINALIZE; @@ -133,8 +133,8 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) /* we are done with the template, now let's convert the result into a * "real" (usable) string and discard the helper structures. */ - CHKiRet(rsCStrFinish(pCStr)); - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pVal, 0)); + CHKiRet(cstrFinalize(pCStr)); + CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pVal, 0)); finalize_it: *ppSz = (iRet == RS_RET_OK) ? pVal : NULL; @@ -272,21 +272,21 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM p = *pp; iLen = *pLen; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); while(*p) { if(*p == '\'') { - CHKiRet(rsCStrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); iLen++; /* reflect the extra character */ } else if((escapeMode == 1) && (*p == '\\')) { - CHKiRet(rsCStrAppendChar(pStrB, '\\')); + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); if(*pbMustBeFreed) free(*pp); /* discard previous value */ @@ -299,7 +299,7 @@ finalize_it: if(iRet != RS_RET_OK) { doSQLEmergencyEscape(*pp, escapeMode); if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -376,7 +376,7 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) p = *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; rsCStrSetAllocIncrement(pStrB, 32); /* process the message and expand escapes @@ -387,22 +387,22 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) switch(*++p) { case '\0': /* the best we can do - it's invalid anyhow... */ - rsCStrAppendChar(pStrB, *p); + cstrAppendChar(pStrB, *p); break; case 'n': - rsCStrAppendChar(pStrB, '\n'); + cstrAppendChar(pStrB, '\n'); ++p; break; case 'r': - rsCStrAppendChar(pStrB, '\r'); + cstrAppendChar(pStrB, '\r'); ++p; break; case '\\': - rsCStrAppendChar(pStrB, '\\'); + cstrAppendChar(pStrB, '\\'); ++p; break; case '%': - rsCStrAppendChar(pStrB, '%'); + cstrAppendChar(pStrB, '%'); ++p; break; case '0': /* numerical escape sequence */ @@ -419,15 +419,15 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) while(*p && isdigit((int)*p)) { i = i * 10 + *p++ - '0'; } - rsCStrAppendChar(pStrB, i); + cstrAppendChar(pStrB, i); break; default: - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); break; } } else - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); } if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -438,14 +438,14 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) return 1; } pTpe->eEntryType = CONSTANT; - rsCStrFinish(pStrB); + cstrFinalize(pStrB); /* We obtain the length from the counted string object * (before we delete it). Later we might take additional * benefit from the counted string object. * 2005-09-09 rgerhards */ pTpe->data.constant.iLenConstant = rsCStrLen(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) + if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) return 1; *pp = p; @@ -552,7 +552,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) p = (unsigned char*) *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -563,13 +563,13 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) pTpe->eEntryType = FIELD; while(*p && *p != '%' && *p != ':') { - rsCStrAppendChar(pStrB, tolower(*p)); + cstrAppendChar(pStrB, tolower(*p)); ++p; /* do NOT do this in tolower()! */ } /* got the name*/ - rsCStrFinish(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) + cstrFinalize(pStrB); + if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) return 1; /* Check frompos, if it has an R, then topos should be a regex */ diff --git a/tools/syslogd.c b/tools/syslogd.c index a9f1dbdb..c224087d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -553,7 +553,7 @@ static char **crunch_list(char *list) #if 0 count=0; while (result[count]) - dbgprintf("#%d: %s\n", count, StripDomains[count++]); + DBGPRINTF("#%d: %s\n", count, StripDomains[count++]); #endif return result; } @@ -780,7 +780,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * rgerhards, 2007-09-14 */ if(*(msg + len - 1) == '\0') { - dbgprintf("dropped NUL at very end of message\n"); + DBGPRINTF("dropped NUL at very end of message\n"); len--; } @@ -790,7 +790,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * turn on/off this handling. rgerhards, 2007-07-23 */ if(bDropTrailingLF && *(msg + len - 1) == '\n') { - dbgprintf("dropped LF at very end of message (DropTrailingLF is set)\n"); + DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); len--; } @@ -815,7 +815,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla iLenDefBuf = iMaxLine; CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); - dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", + DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, len-1); /* Now check if the uncompression worked. If not, there is not much we can do. In * that case, we log an error message but ignore the message itself. Storing the @@ -861,7 +861,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * (I couldn't do any more smart things anyway...). * rgerhards, 2007-9-20 */ - dbgprintf("internal error: iMsg > max msg size in printchopped()\n"); + DBGPRINTF("internal error: iMsg > max msg size in printchopped()\n"); } FINALIZE; /* in this case, we are done... nothing left we can do */ } @@ -1025,14 +1025,14 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { /* not equal, so we are already done... */ - dbgprintf("hostname filter '+%s' does not match '%s'\n", + DBGPRINTF("hostname filter '+%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); FINALIZE; } } else { /* must be -hostname */ if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { /* not equal, so we are already done... */ - dbgprintf("hostname filter '-%s' does not match '%s'\n", + DBGPRINTF("hostname filter '-%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); FINALIZE; } @@ -1053,7 +1053,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce if((!bEqv && !bInv) || (bEqv && bInv)) { /* not equal or inverted selection, so we are already done... */ - dbgprintf("programname filter '%s' does not match '%s'\n", + DBGPRINTF("programname filter '%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); FINALIZE; } @@ -1063,7 +1063,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce if(f->f_filter_type == FILTER_PRI) { /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ + if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) bRet = 0; else @@ -1074,7 +1074,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce CHKiRet(vm.SetMsg(pVM, pMsg)); CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + DBGPRINTF("result of expression evaluation: %lld\n", pResult->val.num); /* VM is destructed on function exit */ bRet = (pResult->val.num) ? 1 : 0; } else { @@ -1166,7 +1166,7 @@ DEFFUNC_llExecFunc(processMsgDoActions) assert(pAction != NULL); if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - dbgprintf("not calling action because the previous one is not suspended\n"); + DBGPRINTF("not calling action because the previous one is not suspended\n"); ABORT_FINALIZE(RS_RET_OK); } @@ -1399,7 +1399,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); } } else { - dbgprintf("no TIMESTAMP detected!\n"); + DBGPRINTF("no TIMESTAMP detected!\n"); bContParse = 0; } @@ -1561,7 +1561,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) /* indeed, this smells like a TAG, so lets use it for this. We take * the HOSTNAME from the sender system instead. */ - dbgprintf("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); + DBGPRINTF("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); moveHOSTNAMEtoTAG(pMsg); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } @@ -1586,22 +1586,21 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the records: the code is currently clean, but we could optimize it! */ if(!bTAGCharDetected) { uchar *pszTAG; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; rsCStrSetAllocIncrement(pStrB, 33); pWork = pBuf; iCnt = 0; while(*p2parse && *p2parse != ':' && *p2parse != ' ') { - rsCStrAppendChar(pStrB, *p2parse++); + cstrAppendChar(pStrB, *p2parse++); ++iCnt; } if(*p2parse == ':') { ++p2parse; - rsCStrAppendChar(pStrB, ':'); + cstrAppendChar(pStrB, ':'); } - rsCStrFinish(pStrB); - - rsCStrConvSzStrAndDestruct(pStrB, &pszTAG, 1); + cstrFinalize(pStrB); + cstrConvSzStrAndDestruct(pStrB, &pszTAG, 1); if(pszTAG == NULL) { /* rger, 2005-11-10: no TAG found - this implies that what * we have considered to be the HOSTNAME is most probably the @@ -1611,7 +1610,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the hostname. This situation is the standard case with * stock BSD syslogd. */ - dbgprintf("No TAG in message, assuming that HOSTNAME is missing.\n"); + DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n"); moveHOSTNAMEtoTAG(pMsg); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } else { /* we have a TAG, so we can happily set it ;) */ @@ -1627,7 +1626,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) */ if(!(flags & INTERNAL_MSG)) { - dbgprintf("HOSTNAME and TAG not parsed by user configuraton.\n"); + DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } } @@ -1688,7 +1687,7 @@ logmsg(msg_t *pMsg, int flags) assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); msg = (char*) pMsg->pszUxTradMsg; - dbgprintf("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); + DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have * a traditional syslog message or one formatted according to syslog-protocol. @@ -1696,14 +1695,14 @@ logmsg(msg_t *pMsg, int flags) * -protocol VERSION field for the detection. */ if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); + DBGPRINTF("Message has syslog-protocol format.\n"); setProtocolVersion(pMsg, 1); if(parseRFCSyslogMsg(pMsg, flags) == 1) { msgDestruct(&pMsg); return; } } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); + DBGPRINTF("Message has legacy syslog format.\n"); setProtocolVersion(pMsg, 0); if(parseLegacySyslogMsg(pMsg, flags) == 1) { msgDestruct(&pMsg); @@ -1753,7 +1752,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) * in an acceptable way. -- rgerhards, 2008-09-16 */ if (pAction->f_prevcount && time(NULL) >= REPEATTIME(pAction)) { - dbgprintf("flush %s: repeated %d times, %d sec.\n", + DBGPRINTF("flush %s: repeated %d times, %d sec.\n", module.GetStateName(pAction->pMod), pAction->f_prevcount, repeatinterval[pAction->f_repeatcount]); actionWriteToAction(pAction); @@ -1789,9 +1788,9 @@ static void debug_switch() if(debugging_on == 0) { debugging_on = 1; - dbgprintf("Switching debugging_on to true\n"); + DBGPRINTF("Switching debugging_on to true\n"); } else { - dbgprintf("Switching debugging_on to false\n"); + DBGPRINTF("Switching debugging_on to false\n"); debugging_on = 0; } @@ -1935,7 +1934,7 @@ void legacyOptsParseTCP(char ch, char *arg) * a minimal delay, but it is much cleaner than the approach of doing everything * inside the signal handler. * rgerhards, 2005-10-26 - * Note: we do not call dbgprintf() as this may cause us to block in case something + * Note: we do not call DBGPRINTF() as this may cause us to block in case something * with the threading is wrong. */ static void doDie(int sig) @@ -1983,7 +1982,7 @@ die(int sig) { char buf[256]; - dbgprintf("exiting on signal %d\n", sig); + DBGPRINTF("exiting on signal %d\n", sig); /* IMPORTANT: we should close the inputs first, and THEN send our termination * message. If we do it the other way around, logmsgInternal() may block on @@ -1998,7 +1997,7 @@ die(int sig) */ /* close the inputs */ - dbgprintf("Terminating input threads...\n"); + DBGPRINTF("Terminating input threads...\n"); thrdTerminateAll(); /* and THEN send the termination log message (see long comment above) */ @@ -2012,17 +2011,17 @@ die(int sig) } /* drain queue (if configured so) and stop main queue worker thread pool */ - dbgprintf("Terminating main queue...\n"); + DBGPRINTF("Terminating main queue...\n"); qqueueDestruct(&pMsgQueue); pMsgQueue = NULL; /* Free ressources and close connections. This includes flushing any remaining * repeated msgs. */ - dbgprintf("Terminating outputs...\n"); + DBGPRINTF("Terminating outputs...\n"); freeSelectors(); - dbgprintf("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); + DBGPRINTF("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); /* rger 2005-02-22 * now clean up the in-memory structures. OK, the OS * would also take care of that, but if we do it @@ -2060,7 +2059,7 @@ die(int sig) */ module.UnloadAndDestructAll(eMOD_LINK_ALL); - dbgprintf("Clean shutdown completed, bye\n"); + DBGPRINTF("Clean shutdown completed, bye\n"); /* dbgClassExit MUST be the last one, because it de-inits the debug system */ dbgClassExit(); @@ -2117,7 +2116,7 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) iFiles, errStr, (long) maxFiles.rlim_max); ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); } - dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); + DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); finalize_it: RETiRet; @@ -2128,7 +2127,7 @@ finalize_it: static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { umask(iUmask); - dbgprintf("umask set to 0%3.3o.\n", iUmask); + DBGPRINTF("umask set to 0%3.3o.\n", iUmask); return RS_RET_OK; } @@ -2213,7 +2212,7 @@ static void freeSelectors(void) selector_t *fPrev; if(Files != NULL) { - dbgprintf("Freeing log structures.\n"); + DBGPRINTF("Freeing log structures.\n"); for(f = Files ; f != NULL ; f = f->f_next) { llExecFunc(&f->llActList, freeSelectorsActions, NULL); @@ -2420,7 +2419,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) { DEFiRet; iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); + DBGPRINTF("\n"); RETiRet; } @@ -2437,43 +2436,44 @@ static void dbgPrintInitInfo(void) int iSelNbr = 1; int i; - dbgprintf("\nActive selectors:\n"); + DBGPRINTF("\nActive selectors:\n"); for (f = Files; f != NULL ; f = f->f_next) { - dbgprintf("Selector %d:\n", iSelNbr++); + DBGPRINTF("Selector %d:\n", iSelNbr++); if(f->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); + DBGPRINTF("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); if(f->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", + DBGPRINTF("hostname: %s '%s'\n", f->eHostnameCmpMode == HN_COMP_MATCH ? "only" : "allbut", rsCStrGetSzStrNoNULL(f->pCSHostnameComp)); if(f->f_filter_type == FILTER_PRI) { for (i = 0; i <= LOG_NFACILITIES; i++) - if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", f->f_filterData.f_pmask[i]); + if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) { + DBGPRINTF(" X "); + } else { + DBGPRINTF("%2X ", f->f_filterData.f_pmask[i]); + } } else if(f->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); + DBGPRINTF("EXPRESSION-BASED Filter: can currently not be displayed"); } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", + DBGPRINTF("PROPERTY-BASED Filter:\n"); + DBGPRINTF("\tProperty.: '%s'\n", rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName)); - dbgprintf("\tOperation: "); + DBGPRINTF("\tOperation: "); if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", + DBGPRINTF("NOT "); + DBGPRINTF("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); + DBGPRINTF("\tValue....: '%s'\n", rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue)); - dbgprintf("\tAction...: "); + DBGPRINTF("\tAction...: "); } - dbgprintf("\nActions:\n"); + DBGPRINTF("\nActions:\n"); llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - dbgprintf("\n"); + DBGPRINTF("\n"); } - dbgprintf("\n"); + DBGPRINTF("\n"); if(bDebugPrintTemplateList) tplPrintList(); if(bDebugPrintModuleList) @@ -2483,24 +2483,24 @@ static void dbgPrintInitInfo(void) if(bDebugPrintCfSysLineHandlerList) dbgPrintCfSysLineHandlers(); - dbgprintf("Messages with malicious PTR DNS Records are %sdropped.\n", + DBGPRINTF("Messages with malicious PTR DNS Records are %sdropped.\n", glbl.GetDropMalPTRMsgs() ? "" : "not "); - dbgprintf("Control characters are %sreplaced upon reception.\n", + DBGPRINTF("Control characters are %sreplaced upon reception.\n", bEscapeCCOnRcv? "" : "not "); if(bEscapeCCOnRcv) - dbgprintf("Control character escape sequence prefix is '%c'.\n", + DBGPRINTF("Control character escape sequence prefix is '%c'.\n", cCCEscapeChar); - dbgprintf("Main queue size %d messages.\n", iMainMsgQueueSize); - dbgprintf("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", + DBGPRINTF("Main queue size %d messages.\n", iMainMsgQueueSize); + DBGPRINTF("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", iMainMsgQueueNumWorkers, iMainMsgQtoWrkShutdown, iMainMsgQPersistUpdCnt); - dbgprintf("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", + DBGPRINTF("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", iMainMsgQtoQShutdown, iMainMsgQtoActShutdown, iMainMsgQtoEnq); - dbgprintf("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", + DBGPRINTF("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", iMainMsgQHighWtrMark, iMainMsgQLowWtrMark, iMainMsgQDiscardMark, iMainMsgQDiscardSeverity); - dbgprintf("Main queue save on shutdown %d, max disk space allowed %lld\n", + DBGPRINTF("Main queue save on shutdown %d, max disk space allowed %lld\n", bMainMsgQSaveOnShutdown, iMainMsgQueMaxDiskSpace); /* TODO: add iActionRetryCount = 0; @@ -2511,7 +2511,7 @@ static void dbgPrintInitInfo(void) setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ - dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); + DBGPRINTF("Work Directory: '%s'.\n", glbl.GetWorkDir()); } @@ -2534,7 +2534,7 @@ startInputModules(void) /* activate here */ thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun); } else { - dbgprintf("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); + DBGPRINTF("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); } pMod = module.GetNxtType(pMod, eMOD_IN); } @@ -2567,11 +2567,11 @@ init(void) pDfltProgNameCmp = NULL; eDfltHostnameCmpMode = HN_NO_COMP; - dbgprintf("rsyslog %s - called init()\n", VERSION); + DBGPRINTF("rsyslog %s - called init()\n", VERSION); /* delete the message queue, which also flushes all messages left over */ if(pMsgQueue != NULL) { - dbgprintf("deleting main message queue\n"); + DBGPRINTF("deleting main message queue\n"); qqueueDestruct(&pMsgQueue); /* delete pThis here! */ pMsgQueue = NULL; } @@ -2582,10 +2582,10 @@ init(void) freeSelectors(); /* Unload all non-static modules */ - dbgprintf("Unloading non-static modules.\n"); + DBGPRINTF("Unloading non-static modules.\n"); module.UnloadAndDestructAll(eMOD_LINK_DYNAMIC_LOADED); - dbgprintf("Clearing templates.\n"); + DBGPRINTF("Clearing templates.\n"); tplDeleteNew(); /* re-setting values to defaults (where applicable) */ @@ -2635,7 +2635,7 @@ init(void) snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); conf.cfline((uchar*)cbuf, &f); } else { - dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); + DBGPRINTF("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } selectorAddList(f); } @@ -2741,7 +2741,7 @@ init(void) } bHaveMainQueue = (MainMsgQueType == QUEUETYPE_DIRECT) ? 0 : 1; - dbgprintf("Main processing queue is initialized and running\n"); + DBGPRINTF("Main processing queue is initialized and running\n"); /* the output part and the queue is now ready to run. So it is a good time * to start the inputs. Please note that the net code above should be @@ -2768,7 +2768,7 @@ init(void) sigAct.sa_handler = sighup_handler; sigaction(SIGHUP, &sigAct, NULL); - dbgprintf(" (re)started.\n"); + DBGPRINTF(" (re)started.\n"); finalize_it: RETiRet; @@ -2799,7 +2799,7 @@ selectorAddList(selector_t *f) selectorDestruct(f); } else { /* successfully created an entry */ - dbgprintf("selector line successfully processed\n"); + DBGPRINTF("selector line successfully processed\n"); /* TODO: we should use the linked list class for the selector list, else we need to add globals * ... well nextp could be added temporarily... * Thanks to varmojfekoj for having the idea to just use "Files" to make this @@ -2831,16 +2831,16 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz if (!strcasecmp((char *) pszType, "fixedarray")) { MainMsgQueType = QUEUETYPE_FIXED_ARRAY; - dbgprintf("main message queue type set to FIXED_ARRAY\n"); + DBGPRINTF("main message queue type set to FIXED_ARRAY\n"); } else if (!strcasecmp((char *) pszType, "linkedlist")) { MainMsgQueType = QUEUETYPE_LINKEDLIST; - dbgprintf("main message queue type set to LINKEDLIST\n"); + DBGPRINTF("main message queue type set to LINKEDLIST\n"); } else if (!strcasecmp((char *) pszType, "disk")) { MainMsgQueType = QUEUETYPE_DISK; - dbgprintf("main message queue type set to DISK\n"); + DBGPRINTF("main message queue type set to DISK\n"); } else if (!strcasecmp((char *) pszType, "direct")) { MainMsgQueType = QUEUETYPE_DIRECT; - dbgprintf("main message queue type set to DIRECT (no queueing at all)\n"); + DBGPRINTF("main message queue type set to DIRECT (no queueing at all)\n"); } else { errmsg.LogError(0, RS_RET_INVALID_PARAMS, "unknown mainmessagequeuetype parameter: %s", (char *) pszType); iRet = RS_RET_INVALID_PARAMS; @@ -3219,7 +3219,7 @@ static rsRetVal mainThread() CHKiRet(init()); if(Debug && debugging_on) { - dbgprintf("Debugging enabled, SIGUSR1 to turn off debugging.\n"); + DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n"); } /* Send a signal to the parent so it can terminate. */ @@ -3251,7 +3251,7 @@ static rsRetVal mainThread() * do the init() and then restart things. * rgerhards, 2005-10-24 */ - dbgprintf("initialization completed, transitioning to regular run mode\n"); + DBGPRINTF("initialization completed, transitioning to regular run mode\n"); mainloop(); @@ -3458,7 +3458,7 @@ doGlblProcessInit(void) if( !(Debug || NoFork) ) { - dbgprintf("Checking pidfile.\n"); + DBGPRINTF("Checking pidfile.\n"); if (!check_pid(PidFile)) { memset(&sigAct, 0, sizeof (sigAct)); @@ -3494,7 +3494,7 @@ doGlblProcessInit(void) } /* tuck my process id away */ - dbgprintf("Writing pidfile %s.\n", PidFile); + DBGPRINTF("Writing pidfile %s.\n", PidFile); if (!check_pid(PidFile)) { if (!write_pid(PidFile)) @@ -3640,7 +3640,7 @@ int realMain(int argc, char **argv) if ((argc -= optind)) usage(); - dbgprintf("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", + DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", VERSION, iCompatibilityMode, glblModPath == NULL ? "" : (char*)glblModPath); /* we are done with the initial option parsing and processing. Now we init the system. */ @@ -3722,7 +3722,7 @@ int realMain(int argc, char **argv) /* END core initializations - we now come back to carrying out command line options*/ while((iRet = bufOptRemove(&ch, &arg)) == RS_RET_OK) { - dbgprintf("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); + DBGPRINTF("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); switch((char)ch) { case '4': glbl.SetDefPFFamily(PF_INET); -- cgit From 74b2b24f508be90d20961304d5e3cce648f3eb7c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 12:31:27 +0200 Subject: removed long-obsoleted property UxTradMsg ... as well as some cleanup (some commented-out code is left to support UxTradMsg again is someone really complains ;)). --- ChangeLog | 1 + doc/property_replacer.html | 4 -- plugins/imfile/imfile.c | 1 - plugins/imklog/imklog.c | 2 - runtime/msg.c | 100 ++++++++++++++++++++++----------------------- runtime/msg.h | 9 ++-- runtime/parser.c | 2 +- tools/syslogd.c | 23 ++++------- 8 files changed, 65 insertions(+), 77 deletions(-) diff --git a/ChangeLog b/ChangeLog index f9e3e32b..beed5663 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- removed long-obsoleted property UxTradMsg - added a generic network stream server (in addition to rather specific syslog tcp server) - added ability for the UDP output action to rebind its send socket after diff --git a/doc/property_replacer.html b/doc/property_replacer.html index a6e9b518..7b604ea0 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -30,10 +30,6 @@ Currently supported are:

    socket. Should be useful for debugging. -uxtradmsg -will disappear soon - do NOT use! - - hostname hostname from the message diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index e3882ce5..927cb82e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -96,7 +96,6 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); - MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 45933436..420ebbf1 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -97,8 +97,6 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsg(pMsg, (char*)msg); - MsgSetUxTradMsg(pMsg, (char*)msg); - MsgSetRawMsg(pMsg, (char*)msg); MsgSetMSG(pMsg, (char*)msg); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); diff --git a/runtime/msg.c b/runtime/msg.c index 55cc48b8..8122383a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -343,7 +343,6 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - free(pThis->pszUxTradMsg); free(pThis->pszRawMsg); free(pThis->pszTAG); free(pThis->pszHOSTNAME); @@ -437,6 +436,10 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + pNew->offAfterPRI = pOld->offAfterPRI; + */ memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); @@ -445,7 +448,6 @@ msg_t* MsgDup(msg_t* pOld) tmpCOPYSZ(FacilityStr); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); - tmpCOPYSZ(UxTradMsg); tmpCOPYSZ(TAG); tmpCOPYSZ(HOSTNAME); tmpCOPYSZ(RcvFrom); @@ -494,10 +496,13 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + objSerializeSCALAR(pStrm, offsAfterPRI, SHORT); + */ objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszMSG, PSZ); - objSerializePTR(pStrm, pszUxTradMsg, PSZ); objSerializePTR(pStrm, pszTAG, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); objSerializePTR(pStrm, pszInputName, PSZ); @@ -701,16 +706,17 @@ static char *getRawMsg(msg_t *pM) return (char*)pM->pszRawMsg; } + +/* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 char *getUxTradMsg(msg_t *pM) { if(pM == NULL) return ""; else - if(pM->pszUxTradMsg == NULL) - return ""; - else - return (char*)pM->pszUxTradMsg; + return (char*)pM->pszRawMsg + pM->offAfterPRI; } +*/ char *getMSG(msg_t *pM) { @@ -724,21 +730,30 @@ char *getMSG(msg_t *pM) } +/* Get PRI value as integer */ +static int getPRIi(msg_t *pM) +{ + assert(pM != NULL); + return (pM->iFacility << 3) + (pM->iSeverity); +} + + /* Get PRI value in text form */ static inline char *getPRI(msg_t *pM) { if(pM == NULL) return ""; - return (char*)pM->bufPRI; -} - + /* there are some cases where bufPRI may not contain a valid string, + * and then we need to build it. + */ + MsgLock(pM); + if(pM->bufPRI[0] == '\0') { + snprintf((char*)pM->bufPRI, sizeof(pM->bufPRI), "<%d>", getPRIi(pM)); + } + MsgUnlock(pM); -/* Get PRI value as integer */ -static int getPRIi(msg_t *pM) -{ - assert(pM != NULL); - return (pM->iFacility << 3) + (pM->iSeverity); + return (char*)pM->bufPRI; } @@ -1017,6 +1032,17 @@ MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl) RETiRet; } +/* set offset after which PRI in raw msg starts + * rgerhards, 2009-06-16 + */ +rsRetVal +MsgSetAfterPRIOffs(msg_t *pMsg, short offs) +{ + assert(pMsg != NULL); + pMsg->offAfterPRI = offs; + return RS_RET_OK; +} + /* rgerhards 2004-11-24: set APP-NAME in msg object * TODO: revisit msg locking code! @@ -1520,40 +1546,6 @@ void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) } -/* Set the UxTradMsg to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetUxTradMsg(). - * rgerhards 2004-11-19 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static void MsgAssignUxTradMsg(msg_t *pMsg, char *pBuf) -{ - assert(pMsg != NULL); - assert(pBuf != NULL); - pMsg->iLenUxTradMsg = strlen(pBuf); - pMsg->pszUxTradMsg = pBuf; -} -#endif - - -/* rgerhards 2004-11-17: set the traditional Unix message in msg object - */ -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg) -{ - assert(pMsg != NULL); - assert(pszUxTradMsg != NULL); - pMsg->iLenUxTradMsg = strlen(pszUxTradMsg); - if(pMsg->pszUxTradMsg != NULL) - free(pMsg->pszUxTradMsg); - if((pMsg->pszUxTradMsg = malloc(pMsg->iLenUxTradMsg + 1)) != NULL) - memcpy(pMsg->pszUxTradMsg, pszUxTradMsg, pMsg->iLenUxTradMsg + 1); - else - dbgprintf("Could not allocate memory for pszUxTradMsg buffer."); - - return(0); -} - - /* rgerhards 2004-11-09: set MSG in msg object */ void MsgSetMSG(msg_t *pMsg, char* pszMSG) @@ -1732,8 +1724,11 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = getMSG(pMsg); } else if(!strcmp((char*) pName, "rawmsg")) { pRes = getRawMsg(pMsg); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 } else if(!strcmp((char*) pName, "uxtradmsg")) { pRes = getUxTradMsg(pMsg); + */ } else if(!strcmp((char*) pName, "inputname")) { pRes = (char*) getInputName(pMsg); } else if(!strcmp((char*) pName, "fromhost")) { @@ -2457,10 +2452,15 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) pThis->msgFlags = pProp->val.num; } else if(isProp("pszRawMsg")) { MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(isProp("offAfterPRI")) { + pThis->offAfterPRI = pProp->val.num; + */ } else if(isProp("pszMSG")) { MsgSetMSG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszUxTradMsg")) { - MsgSetUxTradMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { diff --git a/runtime/msg.h b/runtime/msg.h index 20360641..fe9f87fa 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,7 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; -short bDoLock; /* use the mutex? */ + bool bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ @@ -73,13 +73,13 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - //uchar *pszPRI; /* the PRI as a string */ - uchar bufPRI[5]; + uchar bufPRI[5]; /* PRI as string */ int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we * need to preserve cryptographic verifiers. */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ int iLenRawMsg; /* length of raw message */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ @@ -158,7 +158,7 @@ void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); +rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); @@ -183,6 +183,7 @@ uchar *getRcvFrom(msg_t *pM); #if 0 char *getUxTradMsg(msg_t *pM); +int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); #endif /* The MsgPrepareEnqueue() function is a macro for performance reasons. diff --git a/runtime/parser.c b/runtime/parser.c index 13fb51ec..0b45bfd5 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -314,7 +314,7 @@ rsRetVal parseMsg(msg_t *pMsg) } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - MsgSetUxTradMsg(pMsg, (char*) msg); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); if(pMsg->bParseHOSTNAME == 0) MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); diff --git a/tools/syslogd.c b/tools/syslogd.c index c224087d..2f06d273 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -670,16 +670,9 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) MsgSetHOSTNAME(pMsg, hname); MsgSetRcvFrom(pMsg, hname); + MsgSetAfterPRIOffs(pMsg, p - msg); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); - /* rgerhards 2004-11-19: well, well... we've now seen that we - * have the "hostname problem" also with the traditional Unix - * message. As we like to emulate it, we need to add the hostname - * to it. - */ - if(MsgSetUxTradMsg(pMsg, (char*)p) != 0) - ABORT_FINALIZE(RS_RET_ERR); - logmsg(pMsg, flags); finalize_it: @@ -948,7 +941,6 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) CHKiRet(msgConstruct(&pMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); - MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); @@ -1370,8 +1362,8 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ /* do a sanity check on the version and eat it */ assert(p2parse[0] == '1' && p2parse[1] == ' '); @@ -1471,8 +1463,8 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ /* Check to see if msg contains a timestamp. We start by assuming * that the message timestamp is the time of reciption (which we @@ -1685,8 +1677,9 @@ logmsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - msg = (char*) pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + + msg = (char*) pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have -- cgit From f45dfe93af0c7f5598a8ea953e6b2b82a5427a7c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 13:00:04 +0200 Subject: minor cleanup --- runtime/rsyslog.h | 1 - tcps_sess.c | 2 -- 2 files changed, 3 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 0fafd700..e81c9eef 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -99,7 +99,6 @@ typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ -typedef short bool; #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ diff --git a/tcps_sess.c b/tcps_sess.c index b8ea0878..62882f5b 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -56,8 +56,6 @@ DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ -#define TIME_REQUERY_DFLT 16 // TODO change back! 2 -static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ static int iNbrTimeUsed = 0; /* how often has previous time been used so far? */ -- cgit From aef1a38fe8c7472362904b2f90c67113b21034ab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 13:28:18 +0200 Subject: fixing problems that occurred during the last merge --- tcps_sess.c | 2 - tools/syslogd.c | 291 -------------------------------------------------------- 2 files changed, 293 deletions(-) diff --git a/tcps_sess.c b/tcps_sess.c index 62882f5b..f887e702 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -526,7 +526,5 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, tcps_sessConstructFinalize); ENDObjClassInit(tcps_sess) - - /* vim:set ai: */ diff --git a/tools/syslogd.c b/tools/syslogd.c index fd4bc937..ea8eff7a 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -925,233 +925,6 @@ finalize_it: RETiRet; } -/* This functions looks at the given message and checks if it matches the - * provided filter condition. If so, it returns true, else it returns - * false. This is a helper to logmsg() and meant to drive the decision - * process if a message is to be processed or not. As I expect this - * decision code to grow more complex over time AND logmsg() is already - * a very lengthy function, I thought a separate function is more appropriate. - * 2005-09-19 rgerhards - * 2008-02-25 rgerhards: changed interface, now utilizes iRet, bProcessMsg - * returns is message should be procesed. - */ -static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProcessMsg) -{ - DEFiRet; - unsigned short pbMustBeFreed; - char *pszPropVal; - int bRet = 0; - vm_t *pVM = NULL; - var_t *pResult = NULL; - - assert(f != NULL); - assert(pMsg != NULL); - - /* we first have a look at the global, BSD-style block filters (for tag - * and host). Only if they match, we evaluate the actual filter. - * rgerhards, 2005-10-18 - */ - if(f->eHostnameCmpMode == HN_NO_COMP) { - /* EMPTY BY INTENSION - we check this value first, because - * it is the one most often used, so this saves us time! - */ - } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { - if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - DBGPRINTF("hostname filter '+%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } else { /* must be -hostname */ - if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - DBGPRINTF("hostname filter '-%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } - - if(f->pCSProgNameComp != NULL) { - int bInv = 0, bEqv = 0, offset = 0; - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp)) == '-') { - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp) + 1) == '-') - offset = 1; - else { - bInv = 1; - offset = 1; - } - } - if(!rsCStrOffsetSzStrCmp(f->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) - bEqv = 1; - - if((!bEqv && !bInv) || (bEqv && bInv)) { - /* not equal or inverted selection, so we are already done... */ - DBGPRINTF("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); - FINALIZE; - } - } - - /* done with the BSD-style block filters */ - - if(f->f_filter_type == FILTER_PRI) { - /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || - ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - } else if(f->f_filter_type == FILTER_EXPR) { - CHKiRet(vm.Construct(&pVM)); - CHKiRet(vm.ConstructFinalize(pVM)); - CHKiRet(vm.SetMsg(pVM, pMsg)); - CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); - CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - DBGPRINTF("result of expression evaluation: %lld\n", pResult->val.num); - /* VM is destructed on function exit */ - bRet = (pResult->val.num) ? 1 : 0; - } else { - assert(f->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, f->f_filterData.prop.pCSPropName, &pbMustBeFreed); - - /* Now do the compares (short list currently ;)) */ - switch(f->f_filterData.prop.operation ) { - case FIOP_CONTAINS: - if(rsCStrLocateInSzStr(f->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) - bRet = 1; - break; - case FIOP_ISEQUAL: - if(rsCStrSzStrCmp(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_STARTSWITH: - if(rsCStrSzStrStartsWithCStr(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - case FIOP_EREREGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - default: - /* here, it handles NOP (for performance reasons) */ - assert(f->f_filterData.prop.operation == FIOP_NOP); - bRet = 1; /* as good as any other default ;) */ - break; - } - - /* now check if the value must be negated */ - if(f->f_filterData.prop.isNegated) - bRet = (bRet == 1) ? 0 : 1; - - if(Debug) { - dbgprintf("Filter: check for property '%s' (value '%s') ", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName), - pszPropVal); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("%s '%s': %s\n", - getFIOPName(f->f_filterData.prop.operation), - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue), - bRet ? "TRUE" : "FALSE"); - } - - /* cleanup */ - if(pbMustBeFreed) - free(pszPropVal); - } - -finalize_it: - /* destruct in any case, not just on error, but it makes error handling much easier */ - if(pVM != NULL) - vm.Destruct(&pVM); - - if(pResult != NULL) - var.Destruct(&pResult); - - *bProcessMsg = bRet; - RETiRet; -} - - -/* helper to processMsg(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -typedef struct processMsgDoActions_s { - int bPrevWasSuspended; /* was the previous action suspended? */ - msg_t *pMsg; -} processMsgDoActions_t; -DEFFUNC_llExecFunc(processMsgDoActions) -{ - DEFiRet; - rsRetVal iRetMod; /* return value of module - we do not always pass that back */ - action_t *pAction = (action_t*) pData; - processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; - - assert(pAction != NULL); - - if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - DBGPRINTF("not calling action because the previous one is not suspended\n"); - ABORT_FINALIZE(RS_RET_OK); - } - - iRetMod = actionCallAction(pAction, pDoActData->pMsg); - if(iRetMod == RS_RET_DISCARDMSG) { - ABORT_FINALIZE(RS_RET_DISCARDMSG); - } else if(iRetMod == RS_RET_SUSPENDED) { - /* indicate suspension for next module to be called */ - pDoActData->bPrevWasSuspended = 1; - } else { - pDoActData->bPrevWasSuspended = 0; - } - -finalize_it: - RETiRet; -} - - -/* Process (consume) a received message. Calls the actions configured. - * rgerhards, 2005-10-13 - */ -static void -processMsg(msg_t *pMsg) -{ - selector_t *f; - int bContinue; - int bProcessMsg; - processMsgDoActions_t DoActData; - rsRetVal iRet; - - BEGINfunc - assert(pMsg != NULL); - - /* log the message to the particular outputs */ - - bContinue = 1; - for (f = Files; f != NULL && bContinue ; f = f->f_next) { - /* first check the filters... */ - iRet = shouldProcessThisMessage(f, pMsg, &bProcessMsg); - if(!bProcessMsg) { - continue; - } - - /* ok -- from here, we have action-specific code, nothing really selector-specific -- rger 2007-08-01 */ - DoActData.pMsg = pMsg; - DoActData.bPrevWasSuspended = 0; - if(llExecFunc(&f->llActList, processMsgDoActions, (void*)&DoActData) == RS_RET_DISCARDMSG) - bContinue = 0; - } - ENDfunc -} - /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE @@ -2118,56 +1891,6 @@ static void doDropPrivUid(int iUid) } -/* helper to freeSelectors(), used with llExecFunc() to flush - * pending output. -- rgerhards, 2007-08-02 - * We do not need to lock the action object here as the processing - * queue is already empty and no other threads are running when - * we call this function. -- rgerhards, 2007-12-12 - */ -DEFFUNC_llExecFunc(freeSelectorsActions) -{ - action_t *pAction = (action_t*) pData; - - assert(pAction != NULL); - - /* flush any pending output */ - if(pAction->f_prevcount) { - actionWriteToAction(pAction); - } - - return RS_RET_OK; /* never fails ;) */ -} - - -/* Close all open log files and free selector descriptor array. - */ -static void freeSelectors(void) -{ - selector_t *f; - selector_t *fPrev; - - if(Files != NULL) { - DBGPRINTF("Freeing log structures.\n"); - - for(f = Files ; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, freeSelectorsActions, NULL); - } - - /* actions flushed and ready for destruction - so do that... */ - f = Files; - while (f != NULL) { - fPrev = f; - f = f->f_next; - selectorDestruct(fPrev); - } - - /* Reflect the deletion of the selectors linked list. */ - Files = NULL; - bHaveMainQueue = 0; - } -} - - /* helper to generateConfigDAG, to print out all actions via * the llExecFunc() facility. * rgerhards, 2007-08-02 @@ -2349,20 +2072,6 @@ finalize_it: } -/* helper to dbPrintInitInfo, to print out all actions via - * the llExecFunc() facility. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(dbgPrintInitInfoAction) -{ - DEFiRet; - iRet = actionDbgPrint((action_t*) pData); - DBGPRINTF("\n"); - - RETiRet; -} - - /* print debug information as part of init(). This pretty much * outputs the whole config of rsyslogd. I've moved this code * out of init() to clean it somewhat up. -- cgit