From 31344728fe6f83d4f02ce0e5868c331b4e25d659 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 20 Dec 2008 12:56:41 +0100 Subject: bufgix: $PreserveFQDN was not properly handled for locally emitted messages --- ChangeLog | 2 ++ runtime/glbl.c | 27 ++++++++++++++++++++++++++- runtime/glbl.h | 1 + tools/syslogd.c | 5 ++++- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5056f68d..120ef935 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- bufgix: $PreserveFQDN was not properly handled for locally emitted + messages --------------------------------------------------------------------------- Version 4.1.3 [DEVEL] (rgerhards), 2008-12-17 - added $InputTCPServerAddtlFrameDelimiter config directive, which diff --git a/runtime/glbl.c b/runtime/glbl.c index d06c88ff..28f14320 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -60,6 +60,7 @@ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records static int option_DisallowWarning = 1; /* complain if message from disallowed sender is received */ static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup */ +static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalDomain; /* our local domain name - read-only after startup */ static char **StripDomains = NULL;/* these domains may be stripped before writing logs - r/o after s.u., never touched by init */ static char **LocalHosts = NULL;/* these hosts are logged with their hostname - read-only after startup, never touched by init */ @@ -100,6 +101,7 @@ SIMP_PROP(LocalDomain, LocalDomain, uchar*) SIMP_PROP(StripDomains, StripDomains, char**) SIMP_PROP(LocalHosts, LocalHosts, char**) +SIMP_PROP_SET(LocalFQDNName, LocalFQDNName, uchar*) SIMP_PROP_SET(LocalHostName, LocalHostName, uchar*) SIMP_PROP_SET(DfltNetstrmDrvr, pszDfltNetstrmDrvr, uchar*) /* TODO: use custom function which frees existing value */ SIMP_PROP_SET(DfltNetstrmDrvrCAF, pszDfltNetstrmDrvrCAF, uchar*) /* TODO: use custom function which frees existing value */ @@ -116,7 +118,27 @@ SIMP_PROP_SET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) /* TO static uchar* GetLocalHostName(void) { - return(LocalHostName == NULL ? (uchar*) "[localhost]" : LocalHostName); + uchar *pszRet; + + if(LocalHostName == NULL) + pszRet = (uchar*) "[localhost]"; + else { + if(GetPreserveFQDN() == 1) + pszRet = LocalFQDNName; + else + pszRet = LocalHostName; + } + return(pszRet); +} + + +/* return the current localhost name as FQDN (requires FQDN to be set) + * TODO: we should set the FQDN ourselfs in here! + */ +static uchar* +GetLocalFQDNName(void) +{ + return(LocalFQDNName == NULL ? (uchar*) "[localhost]" : LocalFQDNName); } @@ -186,6 +208,7 @@ CODESTARTobjQueryInterface(glbl) SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); SIMP_PROP(DisableDNS); + SIMP_PROP(LocalFQDNName) SIMP_PROP(LocalHostName) SIMP_PROP(LocalDomain) SIMP_PROP(StripDomains) @@ -270,6 +293,8 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(pszWorkDir); if(LocalHostName != NULL) free(LocalHostName); + if(LocalFQDNName != NULL) + free(LocalFQDNName); ENDObjClassExit(glbl) /* vi:set ai: diff --git a/runtime/glbl.h b/runtime/glbl.h index 205a5212..5bdf4f57 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -48,6 +48,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) SIMP_PROP(DisableDNS, int) + SIMP_PROP(LocalFQDNName, uchar*) SIMP_PROP(LocalHostName, uchar*) SIMP_PROP(LocalDomain, uchar*) SIMP_PROP(StripDomains, char**) diff --git a/tools/syslogd.c b/tools/syslogd.c index 138bdfd8..2cac8fe4 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3288,6 +3288,7 @@ int realMain(int argc, char **argv) uchar legacyConfLine[80]; uchar *LocalHostName; uchar *LocalDomain; + uchar *LocalFQDNName; /* first, parse the command line options. We do not carry out any actual work, just * see what we should do. This relieves us from certain anomalies and we can process @@ -3392,7 +3393,9 @@ int realMain(int argc, char **argv) /* get our host and domain names - we need to do this early as we may emit * error log messages, which need the correct hostname. -- rgerhards, 2008-04-04 */ - net.getLocalHostname(&LocalHostName); + net.getLocalHostname(&LocalFQDNName); + CHKmalloc(LocalHostName = (uchar*) strdup((char*)LocalFQDNName)); + glbl.SetLocalFQDNName(LocalFQDNName); /* set the FQDN before we modify it */ if((p = (uchar*)strchr((char*)LocalHostName, '.'))) { *p++ = '\0'; LocalDomain = p; -- cgit From de1da2c06aaae81fe44cb7d8df38931bc210d05a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 23 Dec 2007 11:43:55 +0100 Subject: doc bugfix: duplicate and invalid link to regex check tool --- doc/manual.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual.html b/doc/manual.html index e8842de6..bc4c0bc5 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -36,9 +36,8 @@ the links below for the

Where <size_nbr> is specified above, diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index b01dd98b..f3771237 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -84,7 +84,7 @@ onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, s { DEFiRet; parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp"); + NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL); RETiRet; } diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index f4830a47..1865d777 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -40,6 +40,7 @@ #include "srUtils.h" #include "errmsg.h" #include "glbl.h" +#include "datetime.h" MODULE_TYPE_INPUT @@ -50,6 +51,7 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) +DEFobjCurrIf(datetime) static int iMaxLine; /* maximum UDP message size supported */ static int *udpLstnSocks = NULL; /* Internet datagram sockets, first element is nbr of elements @@ -59,6 +61,8 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * it so that we can check available memory in willRun() and request * termination if we can not get it. -- rgerhards, 2007-12-27 */ +#define TIME_REQUERY_DFLT 2 +static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ /* config settings */ @@ -141,6 +145,8 @@ BEGINrunInput uchar fromHostIP[NI_MAXHOST]; uchar fromHostFQDN[NI_MAXHOST]; ssize_t l; + struct syslogTime stTime; + int iNbrTimeUsed; CODESTARTrunInput /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, @@ -179,6 +185,7 @@ CODESTARTrunInput for (i = 0; nfds && i < *udpLstnSocks; i++) { if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { socklen = sizeof(frominet); + iNbrTimeUsed = 0; do { /* we now try to read from the file descriptor until there * is no more data. This is done in the hope to get better performance @@ -203,8 +210,11 @@ CODESTARTrunInput */ if(net.isAllowedSender(net.pAllowedSenders_UDP, (struct sockaddr *)&frominet, (char*)fromHostFQDN)) { + if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + datetime.getCurrTime(&stTime); + } parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp"); + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime); } else { dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN); if(glbl.GetOption_DisallowWarning) { @@ -274,6 +284,7 @@ CODESTARTmodExit /* release what we no longer need */ objRelease(errmsg, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); ENDmodExit @@ -293,6 +304,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a net.closeUDPListenSockets(udpLstnSocks); udpLstnSocks = NULL; } + iTimeRequery = TIME_REQUERY_DFLT;/* the default is to query only every second time */ return RS_RET_OK; } @@ -303,6 +315,7 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ @@ -310,9 +323,10 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, + NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit -/* - * vi:set ai: +/* vim:set ai: */ diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 55b8b2df..77d347d9 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -221,7 +221,7 @@ static rsRetVal readSocket(int fd, int iSock) if (iRcvd > 0) { parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock"); + iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); diff --git a/runtime/msg.c b/runtime/msg.c index 69296710..df8c1572 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -239,12 +239,21 @@ rsRetVal MsgEnableThreadSafety(void) /* end locking functions */ -/* "Constructor" for a msg "object". Returns a pointer to +/* This is common code for all Constructors. It is defined in an + * inline'able function so that we can save a function call in the + * actual constructors (otherwise, the msgConstruct would need + * to call msgConstructWithTime(), which would require a + * function call). Now, both can use this inline function. This + * enables us to be optimal, but still have the code just once. * the new object or NULL if no such object could be allocated. * An object constructed via this function should only be destroyed - * via "msgDestruct()". + * via "msgDestruct()". This constructor does not query system time + * itself but rather uses a user-supplied value. This enables the caller + * to do some tricks to save processing time (done, for example, in the + * udp input). + * rgerhards, 2008-10-06 */ -rsRetVal msgConstruct(msg_t **ppThis) +static inline rsRetVal msgBaseConstruct(msg_t **ppThis) { DEFiRet; msg_t *pM; @@ -257,21 +266,58 @@ rsRetVal msgConstruct(msg_t **ppThis) pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; + objConstructSetObjInfo(pM); + + /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ + + *ppThis = pM; + +finalize_it: + RETiRet; +} + + +/* "Constructor" for a msg "object". Returns a pointer to + * the new object or NULL if no such object could be allocated. + * An object constructed via this function should only be destroyed + * via "msgDestruct()". This constructor does not query system time + * itself but rather uses a user-supplied value. This enables the caller + * to do some tricks to save processing time (done, for example, in the + * udp input). + * rgerhards, 2008-10-06 + */ +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime) +{ + DEFiRet; + + CHKiRet(msgBaseConstruct(ppThis)); + memcpy(&(*ppThis)->tRcvdAt, stTime, sizeof(struct syslogTime)); + memcpy(&(*ppThis)->tTIMESTAMP, stTime, sizeof(struct syslogTime)); +finalize_it: + RETiRet; +} + + +/* "Constructor" for a msg "object". Returns a pointer to + * the new object or NULL if no such object could be allocated. + * An object constructed via this function should only be destroyed + * via "msgDestruct()". This constructor, for historical reasons, + * also sets the two timestamps to the current time. + */ +rsRetVal msgConstruct(msg_t **ppThis) +{ + DEFiRet; + + CHKiRet(msgBaseConstruct(ppThis)); /* we initialize both timestamps to contain the current time, so that they * are consistent. Also, this saves us from doing any further time calls just * to obtain a timestamp. The memcpy() should not really make a difference, * especially as I think there is no codepath currently where it would not be * required (after I have cleaned up the pathes ;)). -- rgerhards, 2008-10-02 */ - datetime.getCurrTime(&(pM->tRcvdAt)); - memcpy(&pM->tTIMESTAMP, &pM->tRcvdAt, sizeof(struct syslogTime)); - - objConstructSetObjInfo(pM); - - /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ - - *ppThis = pM; + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + memcpy(&(*ppThis)->tTIMESTAMP, &(*ppThis)->tRcvdAt, sizeof(struct syslogTime)); finalize_it: RETiRet; @@ -2492,7 +2538,5 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) funcDeleteMutex = MsgLockingDummy; funcMsgPrepareEnqueue = MsgLockingDummy; ENDObjClassInit(msg) - -/* - * vi:set ai: +/* vim:set ai: */ diff --git a/runtime/msg.h b/runtime/msg.h index 21cb2c64..d3c0718b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -119,6 +119,7 @@ struct msg { PROTOTYPEObjClassInit(msg); char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); @@ -180,6 +181,5 @@ extern void (*funcMsgPrepareEnqueue)(msg_t *pMsg); #define MsgPrepareEnqueue(pMsg) funcMsgPrepareEnqueue(pMsg) #endif /* #ifndef MSG_H_INCLUDED */ -/* - * vi:set ai: +/* vim:set ai: */ diff --git a/tcps_sess.c b/tcps_sess.c index f5420fc0..7ce9f2f8 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -230,7 +230,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->bAtStrtOfFram = 1; } @@ -313,7 +313,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -324,7 +324,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -343,7 +343,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tools/syslogd.c b/tools/syslogd.c index b576bb6d..a6e17d8f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -581,9 +581,15 @@ void untty(void) * Interface change: added new parameter "InputName", permits the input to provide * a string that identifies it. May be NULL, but must be a valid char* pointer if * non-NULL. + * + * rgerhards, 2008-10-06: + * Interface change: added new parameter "stTime", which enables the caller to provide + * a timestamp that is to be used as timegenerated instead of the current system time. + * This is meant to facilitate performance optimization. Some inputs support such modes. + * If stTime is NULL, the current system time is used. */ -rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName) +static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, + uchar *pszInputName, struct syslogTime *stTime) { DEFiRet; register uchar *p; @@ -591,7 +597,11 @@ rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int msg_t *pMsg; /* Now it is time to create the message object (rgerhards) */ - CHKiRet(msgConstruct(&pMsg)); + if(stTime == NULL) { + CHKiRet(msgConstruct(&pMsg)); + } else { + CHKiRet(msgConstructWithTime(&pMsg, stTime)); + } if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); @@ -684,10 +694,16 @@ finalize_it: * Interface change: added new parameter "InputName", permits the input to provide * a string that identifies it. May be NULL, but must be a valid char* pointer if * non-NULL. + * + * rgerhards, 2008-10-06: + * Interface change: added new parameter "stTime", which enables the caller to provide + * a timestamp that is to be used as timegenerated instead of the current system time. + * This is meant to facilitate performance optimization. Some inputs support such modes. + * If stTime is NULL, the current system time is used. */ rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName) + uchar *pszInputName, struct syslogTime *stTime) { DEFiRet; register int iMsg; @@ -714,9 +730,6 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa * TODO: optimize buffer handling */ iMaxLine = glbl.GetMaxLine(); CHKmalloc(tmpline = malloc(sizeof(uchar) * (iMaxLine + 1))); -# ifdef USE_NETZIP - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); -# endif /* we first check if we have a NUL character at the very end of the * message. This seems to be a frequent problem with a number of senders. @@ -762,6 +775,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ int ret; iLenDefBuf = iMaxLine; + CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, len-1); @@ -800,7 +814,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); } else { /* This case in theory never can happen. If it happens, we have * a logic error. I am checking for it, because if I would not, @@ -852,7 +866,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); finalize_it: if(tmpline != NULL) @@ -1344,6 +1358,13 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags) } else { /* we can not parse, so we get the system we * received the data from. + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); */ MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } -- cgit From 7b3c05da9f126063384c80e9dd6fd5b2ae610074 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 10:28:44 +0200 Subject: simple (yet efficient) name caching added to imudp --- plugins/imudp/imudp.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 1865d777..d13314e8 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -133,6 +133,15 @@ finalize_it: /* This function is called to gather input. * Note that udpLstnSocks must be non-NULL because otherwise we would not have * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 + * rgerhards, 2008-10-07: I have implemented a very simple, yet in most cases probably + * highly efficient "name caching". Before querying a name, I now check if the name to be + * queried is the same as the one queried in the last message processed. If that is the + * case, we can simple re-use the previous value. This algorithm works quite well with + * few sender, especially if they emit messages in bursts. The more sender and the + * more intermixed messages arrive, the less this algorithm works, but the overhead + * is so minimal (a simple memory compare and move) that this does not hurt. Even + * with a real name lookup cache, this optimization here is useful as it is quicker + * than even a cache lookup). */ BEGINrunInput int maxfds; @@ -140,6 +149,7 @@ BEGINrunInput int i; fd_set readfds; struct sockaddr_storage frominet; + struct sockaddr_storage frominetPrev; socklen_t socklen; uchar fromHost[NI_MAXHOST]; uchar fromHostIP[NI_MAXHOST]; @@ -148,6 +158,10 @@ BEGINrunInput struct syslogTime stTime; int iNbrTimeUsed; CODESTARTrunInput + /* start "name caching" algo by making sure the previous system indicator + * is invalidated. + */ + memset(&frominetPrev, 0, sizeof(frominetPrev)); /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, * right into the sleep below. @@ -184,7 +198,7 @@ CODESTARTrunInput for (i = 0; nfds && i < *udpLstnSocks; i++) { if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { - socklen = sizeof(frominet); + socklen = sizeof(frominet); iNbrTimeUsed = 0; do { /* we now try to read from the file descriptor until there @@ -199,7 +213,9 @@ CODESTARTrunInput l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if(l > 0) { - if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) { + if(memcmp(&frominet, &frominetPrev, socklen) == 0 || + net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) { + memcpy(&frominetPrev, &frominet, socklen); dbgprintf("Message from inetd socket: #%d, host: %s\n", udpLstnSocks[i+1], fromHost); /* Here we check if a host is permitted to send us -- cgit From cdecd7e524a5114ccff4f2909b32738bdb33c6ea Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 11:46:46 +0200 Subject: slightly improved lock contention situation by moving out of the critical section what could so with acceptable consequences --- runtime/queue.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 76c2f10f..f5f770b3 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2101,6 +2101,15 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) ISOBJ_TYPE_assert(pThis, queue); + /* first check if we need to discard this message (which will cause CHKiRet() to exit) + * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize + * and bRunsDA parameters may not reflect the correct settings here, but they are + * "good enough" in the sense that they can be used to drive the decision. Valgrind's + * threading tools may point this access to be an error, but this is done + * intentional. I do not see this causes problems to us. + */ + CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + /* Please note that this function is not cancel-safe and consequently * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE * during its execution. If that is not done, race conditions occur if the @@ -2112,9 +2121,6 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) d_pthread_mutex_lock(pThis->mut); } - /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ - CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); - /* then check if we need to add an assistance disk queue */ if(pThis->bIsDA) CHKiRet(queueChkStrtDA(pThis)); -- cgit From 8528344ef58b5d2907bba8809f63d0bca2ce8d38 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 14:26:41 +0200 Subject: "output" timestamp now taken from mesg's time generated This enhances performance and, as some have pointed out, is probably also more consistent with what users expect how the various output-timestamp related function should work. This commit needs some more testing. --- ChangeLog | 9 +++++++++ action.c | 6 +++++- dirty.h | 2 +- plugins/imrelp/imrelp.c | 2 +- plugins/imudp/imudp.c | 6 ++++-- plugins/imuxsock/imuxsock.c | 2 +- runtime/datetime.c | 12 ++++++++++-- runtime/datetime.h | 2 +- runtime/msg.c | 15 +++++++++------ runtime/msg.h | 9 ++++++++- runtime/sysvar.c | 2 +- tcps_sess.c | 8 ++++---- tools/syslogd.c | 17 +++++------------ 13 files changed, 59 insertions(+), 33 deletions(-) diff --git a/ChangeLog b/ChangeLog index 346ff39c..0f3d4591 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-?? + +********************************* WARNING ********************************* +This version has a slightly different on-disk format for message entries. +As a consequence, old queue files being read by this version may have +an invalid output timestamp, which could result to some malfunction inside +the output driver. It is recommended to drain queues with the previous +version before switching to this one. +********************************* WARNING ********************************* + - added $UDPServerTimeRequery option which enables to work with less acurate timestamps in favor of performance. This enables querying of the time only every n-th time if imudp is running in the tight diff --git a/action.c b/action.c index a25eca23..cc62f028 100644 --- a/action.c +++ b/action.c @@ -602,7 +602,7 @@ actionWriteToAction(action_t *pAction) * ... RAWMSG is a problem ... Please note that digital * signatures inside the message are also invalidated. */ - datetime.getCurrTime(&(pMsg->tRcvdAt)); + datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); MsgSetMSG(pMsg, (char*)szRepMsg); MsgSetRawMsg(pMsg, (char*)szRepMsg); @@ -640,6 +640,7 @@ actionWriteToAction(action_t *pAction) * while in the on-disk queue. Also need to think about a few other implications. * rgerhards, 2008-09-17 */ +#if 0 { struct tm tTm; tTm.tm_sec = pAction->f_pMsg->tRcvdAt.second; @@ -655,7 +656,10 @@ actionWriteToAction(action_t *pAction) /* note: mktime seems to do a stat(/etc/localtime), so this is also sub-optimal! */ dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); } +#endif + pAction->f_time = pAction->f_pMsg->ttGenTime; +dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); //pAction->f_time = getActNow(pAction); /* re-init time flags */ /* Note: tLastExec could be set in the if block above, but f_time causes us a hard time * so far, I do not see a solution to getting rid of it. -- rgerhards, 2008-09-16 diff --git a/dirty.h b/dirty.h index 58c4ea39..8e7ffd84 100644 --- a/dirty.h +++ b/dirty.h @@ -40,7 +40,7 @@ rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); -rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime); +rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index f3771237..4515acd7 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -84,7 +84,7 @@ onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, s { DEFiRet; parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL); + NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); RETiRet; } diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index d13314e8..d58a0d8e 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -155,6 +155,7 @@ BEGINrunInput uchar fromHostIP[NI_MAXHOST]; uchar fromHostFQDN[NI_MAXHOST]; ssize_t l; + time_t ttGenTime; struct syslogTime stTime; int iNbrTimeUsed; CODESTARTrunInput @@ -227,10 +228,11 @@ CODESTARTrunInput if(net.isAllowedSender(net.pAllowedSenders_UDP, (struct sockaddr *)&frominet, (char*)fromHostFQDN)) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime); + datetime.getCurrTime(&stTime, &ttGenTime); } parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime); + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", + &stTime, ttGenTime); } else { dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN); if(glbl.GetOption_DisallowWarning) { diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 77d347d9..efa0365d 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -221,7 +221,7 @@ static rsRetVal readSocket(int fd, int iSock) if (iRcvd > 0) { parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL); + iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); diff --git a/runtime/datetime.c b/runtime/datetime.c index 20ca6191..aa1956d7 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -62,9 +62,14 @@ DEFobjCurrIf(errmsg) * most portable and removes the need for additional structures * (but I have to admit it is somewhat "bulky";)). * - * Obviously, all caller-provided pointers must not be NULL... + * Obviously, *t must not be NULL... + * + * rgerhards, 2008-10-07: added ttSeconds to provide a way to + * obtain the second-resolution UNIX timestamp. This is needed + * in some situations to minimize time() calls (namely when doing + * output processing). This can be left NULL if not needed. */ -static void getCurrTime(struct syslogTime *t) +static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) { struct timeval tp; struct tm *tm; @@ -83,6 +88,9 @@ static void getCurrTime(struct syslogTime *t) # else gettimeofday(&tp, NULL); # endif + if(ttSeconds != NULL) + *ttSeconds = tp.tv_sec; + tm = localtime_r((time_t*) &(tp.tv_sec), &tmBuf); t->year = tm->tm_year + 1900; diff --git a/runtime/datetime.h b/runtime/datetime.h index 2210af02..0739588d 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -35,7 +35,7 @@ typedef struct datetime_s { /* interfaces */ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ - void (*getCurrTime)(struct syslogTime *t); + void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, char** ppszTS); rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, char** pszTS); int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst); diff --git a/runtime/msg.c b/runtime/msg.c index df8c1572..fd838591 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -286,11 +286,12 @@ finalize_it: * udp input). * rgerhards, 2008-10-06 */ -rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime) +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; CHKiRet(msgBaseConstruct(ppThis)); + (*ppThis)->ttGenTime = ttGenTime; memcpy(&(*ppThis)->tRcvdAt, stTime, sizeof(struct syslogTime)); memcpy(&(*ppThis)->tTIMESTAMP, stTime, sizeof(struct syslogTime)); @@ -316,7 +317,7 @@ rsRetVal msgConstruct(msg_t **ppThis) * especially as I think there is no codepath currently where it would not be * required (after I have cleaned up the pathes ;)). -- rgerhards, 2008-10-02 */ - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt), &((*ppThis)->ttGenTime)); memcpy(&(*ppThis)->tTIMESTAMP, &(*ppThis)->tRcvdAt, sizeof(struct syslogTime)); finalize_it: @@ -442,7 +443,7 @@ msg_t* MsgDup(msg_t* pOld) assert(pOld != NULL); BEGINfunc - if(msgConstruct(&pNew) != RS_RET_OK) { + if(msgConstructWithTime(&pNew, &pOld->tTIMESTAMP, pOld->ttGenTime) != RS_RET_OK) { return NULL; } @@ -453,8 +454,7 @@ msg_t* MsgDup(msg_t* pOld) pNew->bParseHOSTNAME = pOld->bParseHOSTNAME; pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; - memcpy(&pNew->tRcvdAt, &pOld->tRcvdAt, sizeof(struct syslogTime)); - memcpy(&pNew->tTIMESTAMP, &pOld->tTIMESTAMP, sizeof(struct syslogTime)); + pNew->ttGenTime = pOld->ttGenTime; tmpCOPYSZ(Severity); tmpCOPYSZ(SeverityStr); tmpCOPYSZ(Facility); @@ -508,6 +508,7 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, iSeverity, SHORT); objSerializeSCALAR(pStrm, iFacility, SHORT); objSerializeSCALAR(pStrm, msgFlags, INT); + objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME); @@ -1669,7 +1670,7 @@ static uchar *getNOW(eNOWType eNow) return NULL; } - datetime.getCurrTime(&t); + datetime.getCurrTime(&t, NULL); switch(eNow) { case NOW_NOW: snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day); @@ -2477,6 +2478,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) MsgSetPROCID(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSMSGID")) { MsgSetMSGID(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + } else if(isProp("ttGenTime")) { + pThis->ttGenTime = pProp->val.num; } else if(isProp("tRcvdAt")) { memcpy(&pThis->tRcvdAt, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); } else if(isProp("tTIMESTAMP")) { diff --git a/runtime/msg.h b/runtime/msg.h index d3c0718b..d2fc2f30 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -99,6 +99,13 @@ struct msg { cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. + While this field looks redundant, it is required because a Unix timestamp + is used at later processing stages (namely in the output arena). Thanks to + the subleties of how time is defined, there is no reliable way to reconstruct + the Unix timestamp from the syslogTime fields (in practice, we may be close + enough to reliable, but I prefer to leave the subtle things to the OS, where + it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ @@ -119,7 +126,7 @@ struct msg { PROTOTYPEObjClassInit(msg); char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); -rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime); +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); diff --git a/runtime/sysvar.c b/runtime/sysvar.c index 5eec8f67..c102d1f5 100644 --- a/runtime/sysvar.c +++ b/runtime/sysvar.c @@ -84,7 +84,7 @@ getNOW(eNOWType eNow, cstr_t **ppStr) uchar szBuf[16]; struct syslogTime t; - datetime.getCurrTime(&t); + datetime.getCurrTime(&t, NULL); switch(eNow) { case NOW_NOW: snprintf((char*) szBuf, sizeof(szBuf)/sizeof(uchar), "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day); diff --git a/tcps_sess.c b/tcps_sess.c index 7ce9f2f8..13644f45 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -230,7 +230,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->bAtStrtOfFram = 1; } @@ -313,7 +313,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -324,7 +324,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -343,7 +343,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tools/syslogd.c b/tools/syslogd.c index a6e17d8f..e794e2d1 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -589,7 +589,7 @@ void untty(void) * If stTime is NULL, the current system time is used. */ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime) + uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; register uchar *p; @@ -600,7 +600,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { - CHKiRet(msgConstructWithTime(&pMsg, stTime)); + CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); @@ -703,7 +703,7 @@ finalize_it: */ rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime) + uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; register int iMsg; @@ -814,7 +814,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); } else { /* This case in theory never can happen. If it happens, we have * a logic error. I am checking for it, because if I would not, @@ -866,7 +866,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); finalize_it: if(tmpline != NULL) @@ -1358,13 +1358,6 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags) } else { /* we can not parse, so we get the system we * received the data from. - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); */ MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } -- cgit From 0fa23994669417fff4c4c057ce0c9d1e96f6d56c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 15:10:03 +0200 Subject: cleanup of output timestamp generation --- action.c | 38 ++------------------------------------ runtime/msg.c | 4 ++++ 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/action.c b/action.c index cc62f028..ec8732bc 100644 --- a/action.c +++ b/action.c @@ -630,42 +630,8 @@ actionWriteToAction(action_t *pAction) FINALIZE; } - - - /* TODO: move this to msg object or some other object. This is just for quick testing! - * ALSO, THIS DOES NOT YET WORK PROPERLY! - * The reason is that we do not know the DST status, which is major pain. I need to - * think about obtaining this information (or the actual Unix timestamp) when I - * create the reception timestamp, but that also means I need to preserve that information - * while in the on-disk queue. Also need to think about a few other implications. - * rgerhards, 2008-09-17 - */ -#if 0 - { - struct tm tTm; - tTm.tm_sec = pAction->f_pMsg->tRcvdAt.second; - tTm.tm_min = pAction->f_pMsg->tRcvdAt.minute; - tTm.tm_hour = pAction->f_pMsg->tRcvdAt.hour; - tTm.tm_mday = pAction->f_pMsg->tRcvdAt.day; - tTm.tm_mon = pAction->f_pMsg->tRcvdAt.month - 1; - tTm.tm_year = pAction->f_pMsg->tRcvdAt.year - 1900; - /********************************************************************************/ - tTm.tm_isdst = 1; /* TODO THIS IS JUST VALID FOR THE NEXT FEW DAYS ;) TODO */ - /********************************************************************************/ - pAction->f_time = mktime(&tTm); -/* note: mktime seems to do a stat(/etc/localtime), so this is also sub-optimal! */ -dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); - } -#endif - - pAction->f_time = pAction->f_pMsg->ttGenTime; -dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); - //pAction->f_time = getActNow(pAction); /* re-init time flags */ - /* Note: tLastExec could be set in the if block above, but f_time causes us a hard time - * so far, I do not see a solution to getting rid of it. -- rgerhards, 2008-09-16 - */ - - + /* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */ + pAction->f_time = pAction->f_pMsg->ttGenTime; /* When we reach this point, we have a valid, non-disabled action. * So let's enqueue our message for execution. -- rgerhards, 2007-07-24 diff --git a/runtime/msg.c b/runtime/msg.c index fd838591..c030fa45 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -778,6 +778,7 @@ int getPRIi(msg_t *pM) char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { + BEGINfunc if(pM == NULL) return ""; @@ -849,11 +850,13 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return(pM->pszTIMESTAMP_SecFrac); } + ENDfunc return "INVALID eFmt OPTION!"; } char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) { + BEGINfunc if(pM == NULL) return ""; @@ -925,6 +928,7 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return(pM->pszRcvdAt_SecFrac); } + ENDfunc return "INVALID eFmt OPTION!"; } -- cgit From 82b583c4f99dd9beb30360f222c4d2a1152f75e1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Oct 2008 14:56:02 +0200 Subject: restructured imudp receive loop cleaned up previous code and redid it in a way that makes it much easier to extend it also added a new macro DBGPRINTF which is a performance-optimzed version of dbgprintf --- plugins/imudp/imudp.c | 145 +++++++++++++++++++++++++++++--------------------- runtime/debug.h | 1 + 2 files changed, 86 insertions(+), 60 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index d58a0d8e..7f9afb68 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -130,6 +130,85 @@ finalize_it: } +/* This function is a helper to runInput. I have extracted it + * from the main loop just so that we do not have that large amount of code + * in a single place. This function takes a socket and pulls messages from + * it until the socket does not have any more waiting. + * rgerhards, 2008-01-08 + * We try to read from the file descriptor until there + * is no more data. This is done in the hope to get better performance + * out of the system. However, this also means that a descriptor + * monopolizes processing while it contains data. This can lead to + * data loss in other descriptors. However, if the system is incapable of + * handling the workload, we will loss data in any case. So it doesn't really + * matter where the actual loss occurs - it is always random, because we depend + * on scheduling order. -- rgerhards, 2008-10-02 + */ +static inline rsRetVal +processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, + uchar *fromHost, uchar *fromHostFQDN, uchar *fromHostIP) +{ + DEFiRet; + int iNbrTimeUsed; + time_t ttGenTime; + struct syslogTime stTime; + socklen_t socklen; + ssize_t l; + struct sockaddr_storage frominet; + char errStr[1024]; + + iNbrTimeUsed = 0; + while(1) { /* loop is terminated if we have a bad receive, done below in the body */ + socklen = sizeof(struct sockaddr_storage); + l = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); + if(l < 0) { + if(errno != EINTR && errno != EAGAIN) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr); + errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet"); + } + ABORT_FINALIZE(RS_RET_ERR); + } + + /* if we reach this point, we had a good receive and can process the packet received */ + /* check if we have a different sender than before, if so, we need to query some new values */ + if(memcmp(&frominet, frominetPrev, socklen) != 0) { + CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us + * syslog messages. If it isn't, we do not further + * process the message but log a warning (if we are + * configured to do this). + * rgerhards, 2005-09-26 + */ + *pbIsPermitted = net.isAllowedSender(net.pAllowedSenders_UDP, + (struct sockaddr *)&frominet, (char*)fromHostFQDN); + + if(!*pbIsPermitted) { + DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); + if(glbl.GetOption_DisallowWarning) { + errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded", + (char*)fromHost); + } + } + } + + DBGPRINTF("Message from inetd socket: #%d, host: %s, isPermitted: %d\n", fd, fromHost, *pbIsPermitted); + if(*pbIsPermitted) { + if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + datetime.getCurrTime(&stTime, &ttGenTime); + } + parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime, ttGenTime); + } + } + + +finalize_it: + RETiRet; +} + + /* This function is called to gather input. * Note that udpLstnSocks must be non-NULL because otherwise we would not have * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 @@ -148,20 +227,16 @@ BEGINrunInput int nfds; int i; fd_set readfds; - struct sockaddr_storage frominet; struct sockaddr_storage frominetPrev; - socklen_t socklen; + int bIsPermitted; uchar fromHost[NI_MAXHOST]; uchar fromHostIP[NI_MAXHOST]; uchar fromHostFQDN[NI_MAXHOST]; - ssize_t l; - time_t ttGenTime; - struct syslogTime stTime; - int iNbrTimeUsed; CODESTARTrunInput /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ + bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, @@ -198,61 +273,11 @@ CODESTARTrunInput nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); for (i = 0; nfds && i < *udpLstnSocks; i++) { - if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { - socklen = sizeof(frominet); - iNbrTimeUsed = 0; - do { - /* we now try to read from the file descriptor until there - * is no more data. This is done in the hope to get better performance - * out of the system. However, this also means that a descriptor - * monopolizes processing while it contains data. This can lead to - * data loss in other descriptors. However, if the system is incapable of - * handling the workload, we will loss data in any case. So it doesn't really - * matter where the actual loss occurs - it is always random, because we depend - * on scheduling order. -- rgerhards, 2008-10-02 - */ - l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0, - (struct sockaddr *)&frominet, &socklen); - if(l > 0) { - if(memcmp(&frominet, &frominetPrev, socklen) == 0 || - net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) { - memcpy(&frominetPrev, &frominet, socklen); - dbgprintf("Message from inetd socket: #%d, host: %s\n", - udpLstnSocks[i+1], fromHost); - /* Here we check if a host is permitted to send us - * syslog messages. If it isn't, we do not further - * process the message but log a warning (if we are - * configured to do this). - * rgerhards, 2005-09-26 - */ - if(net.isAllowedSender(net.pAllowedSenders_UDP, - (struct sockaddr *)&frominet, (char*)fromHostFQDN)) { - if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); - } - parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", - &stTime, ttGenTime); - } else { - dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN); - if(glbl.GetOption_DisallowWarning) { - errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded", - (char*)fromHost); - } - } - } - } else if(l < 0 && errno != EINTR && errno != EAGAIN) { - char errStr[1024]; - rs_strerror_r(errno, errStr, sizeof(errStr)); - dbgprintf("INET socket error: %d = %s.\n", errno, errStr); - errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet"); - /* should be harmless */ - sleep(1); - } - } while(l > 0); /* Warning: do ... while()! */ - - --nfds; /* indicate we have processed one descriptor */ + if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { + processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, + fromHost, fromHostFQDN, fromHostIP); } + --nfds; /* indicate we have processed one descriptor */ } /* end of a run, back to loop for next recv() */ } diff --git a/runtime/debug.h b/runtime/debug.h index d9d576b5..7ac29765 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -100,6 +100,7 @@ void dbgSetThrdName(uchar *pszName); void dbgPrintAllDebugInfo(void); /* macros */ +#define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } #ifdef RTINST # define BEGINfunc static dbgFuncDB_t *pdbgFuncDB; int dbgCALLStaCK_POP_POINT = dbgEntrFunc(&pdbgFuncDB, __FILE__, __func__, __LINE__); # define ENDfunc dbgExitFunc(pdbgFuncDB, dbgCALLStaCK_POP_POINT, RS_RET_NO_IRET); -- cgit From ace4f2f75202aec39449dac11b9eb1deca7428d7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Oct 2008 18:55:11 +0200 Subject: reordered imudp processing. Message parsing is now done as part of main message queue worker processing (was part of the input thread) This should also improve performance, as potentially more work is done in parallel. --- ChangeLog | 6 + dirty.h | 3 + plugins/imudp/imudp.c | 27 ++++- runtime/Makefile.am | 2 + runtime/msg.h | 7 +- runtime/parser.c | 314 ++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/parser.h | 30 +++++ runtime/queue.c | 2 - runtime/rsyslog.h | 1 + runtime/wti.c | 6 +- tools/syslogd.c | 18 ++- 11 files changed, 396 insertions(+), 20 deletions(-) create mode 100644 runtime/parser.c create mode 100644 runtime/parser.h diff --git a/ChangeLog b/ChangeLog index 0f3d4591..2f9f34a6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,12 @@ the output driver. It is recommended to drain queues with the previous version before switching to this one. ********************************* WARNING ********************************* +- reordered imudp processing. Message parsing is now done as part of main + message queue worker processing (was part of the input thread) + This should also improve performance, as potentially more work is + done in parallel. +- bugfix: compressed syslog messages could be slightly mis-uncompressed + if the last byte of the compressed record was a NUL - added $UDPServerTimeRequery option which enables to work with less acurate timestamps in favor of performance. This enables querying of the time only every n-th time if imudp is running in the tight diff --git a/dirty.h b/dirty.h index 8e7ffd84..f5d0415e 100644 --- a/dirty.h +++ b/dirty.h @@ -61,6 +61,9 @@ extern int bReduceRepeatMsgs; #define BACKOFF(f) { if (++(f)->f_repeatcount > MAXREPEAT) \ (f)->f_repeatcount = MAXREPEAT; \ } +extern int bDropTrailingLF; +extern uchar cCCEscapeChar; +extern int bEscapeCCOnRcv; #ifdef USE_NETZIP /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 7f9afb68..aab201a9 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -40,6 +40,8 @@ #include "srUtils.h" #include "errmsg.h" #include "glbl.h" +#include "msg.h" +#include "parser.h" #include "datetime.h" MODULE_TYPE_INPUT @@ -153,15 +155,16 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, time_t ttGenTime; struct syslogTime stTime; socklen_t socklen; - ssize_t l; + ssize_t lenRcvBuf; struct sockaddr_storage frominet; + msg_t *pMsg; char errStr[1024]; iNbrTimeUsed = 0; while(1) { /* loop is terminated if we have a bad receive, done below in the body */ socklen = sizeof(struct sockaddr_storage); - l = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); - if(l < 0) { + lenRcvBuf = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); + if(lenRcvBuf < 0) { if(errno != EINTR && errno != EAGAIN) { rs_strerror_r(errno, errStr, sizeof(errStr)); DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr); @@ -193,13 +196,25 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, } } - DBGPRINTF("Message from inetd socket: #%d, host: %s, isPermitted: %d\n", fd, fromHost, *pbIsPermitted); + DBGPRINTF("recv(%d,%d)/%s,acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, fromHost, *pbIsPermitted, pRcvBuf); + if(*pbIsPermitted) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); } - parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime, ttGenTime); + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + /* first trim the buffer to what we have actually received */ + CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); + memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); + pMsg->iLenRawMsg = lenRcvBuf; + MsgSetInputName(pMsg, "imudp"); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + pMsg->bParseHOSTNAME = MSG_PARSE_HOSTNAME; + pMsg->msgFlags = NOFLAG; + MsgSetRcvFrom(pMsg, (char*)fromHost); + CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); + CHKiRet(submitMsg(pMsg)); } } diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 81a9d5bd..7c70dd8d 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -16,6 +16,8 @@ librsyslog_la_SOURCES = \ glbl.c \ conf.c \ conf.h \ + parser.h \ + parser.c \ msg.c \ msg.h \ linkedlist.c \ diff --git a/runtime/msg.h b/runtime/msg.h index d2fc2f30..98635f85 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -52,7 +52,10 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; pthread_mutex_t mut; - int iRefCount; /* reference counter (0 = unused) */ + flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because + once data has entered the queue, this property is no longer needed. */ + short iRefCount; /* reference counter (0 = unused) */ + short bIsParsed; /* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */ short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -60,8 +63,6 @@ struct msg { * sockets. All in all, the parser would need parse templates, that would * resolve all these issues... rgerhards, 2005-10-06 */ - flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because - once data has entered the queue, this property is no longer needed. */ short iSeverity; /* the severity 0..7 */ uchar *pszSeverity; /* severity as string... */ int iLenSeverity; /* ... and its length. */ diff --git a/runtime/parser.c b/runtime/parser.c new file mode 100644 index 00000000..8c4272a0 --- /dev/null +++ b/runtime/parser.c @@ -0,0 +1,314 @@ +/* parser.c + * This module contains functions for message parsers. It still needs to be + * converted into an object (and much extended). + * + * Module begun 2008-10-09 by Rainer Gerhards (based on previous code from syslogd.c) + * + * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" +#include +#include +#include +#ifdef USE_NETZIP +#include +#endif + +#include "rsyslog.h" +#include "dirty.h" +#include "msg.h" +#include "obj.h" +#include "errmsg.h" + +/* some defines */ +#define DEFUPRI (LOG_USER|LOG_NOTICE) + +#warning "msg object must be updated with new property for persisting the queue!" +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) +DEFobjCurrIf(errmsg) + +/* static data */ + + +/* this is a dummy class init + */ +rsRetVal parserClassInit(void) +{ + DEFiRet; + + /* request objects we use */ + CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +// TODO: free components! see action.c +finalize_it: + RETiRet; +} + + +/* uncompress a received message if it is compressed. + * pMsg->pszRawMsg buffer is updated. + * rgerhards, 2008-10-09 + */ +static inline rsRetVal uncompressMessage(msg_t *pMsg) +{ + DEFiRet; +# ifdef USE_NETZIP + uchar *deflateBuf = NULL; + uLongf iLenDefBuf; + uchar *pszMsg; + size_t lenMsg; + + assert(pMsg != NULL); + pszMsg = pMsg->pszRawMsg; + lenMsg = pMsg->iLenRawMsg; + + /* we first need to check if we have a compressed record. If so, + * we must decompress it. + */ + if(lenMsg > 0 && *pszMsg == 'z') { /* compressed data present? (do NOT change order if conditions!) */ + /* we have compressed data, so let's deflate it. We support a maximum + * message size of iMaxLine. If it is larger, an error message is logged + * and the message is dropped. We do NOT try to decompress larger messages + * as such might be used for denial of service. It might happen to later + * builds that such functionality be added as an optional, operator-configurable + * feature. + */ + int ret; + iLenDefBuf = glbl.GetMaxLine(); + CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1))); + ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); + DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", + ret, (long) iLenDefBuf, (int) (lenMsg-1)); + /* Now check if the uncompression worked. If not, there is not much we can do. In + * that case, we log an error message but ignore the message itself. Storing the + * compressed text is dangerous, as it contains control characters. So we do + * not do this. If someone would like to have a copy, this code here could be + * modified to do a hex-dump of the buffer in question. We do not include + * this functionality right now. + * rgerhards, 2006-12-07 + */ + if(ret != Z_OK) { + errmsg.LogError(0, NO_ERRCODE, "Uncompression of a message failed with return code %d " + "- enable debug logging if you need further information. " + "Message ignored.", ret); + FINALIZE; /* unconditional exit, nothing left to do... */ + } + free(pMsg->pszRawMsg); + pMsg->pszRawMsg = deflateBuf; + pMsg->iLenRawMsg = iLenDefBuf; + deflateBuf = NULL; /* logically "freed" - caller is now responsible */ + } +finalize_it: + if(deflateBuf != NULL) + free(deflateBuf); + +# else /* ifdef USE_NETZIP */ + + /* in this case, we still need to check if the message is compressed. If so, we must + * tell the user we can not accept it. + */ + if(len > 0 && *msg == 'z') { + errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " + "support enabled. The message will be ignored."); + ABORT_FINALIZE(RS_RET_NO_ZIP); + } + +# endif /* ifdef USE_NETZIP */ + + RETiRet; +} + + +/* sanitize a received message + * if a message gets to large during sanitization, it is truncated. This is + * as specified in the upcoming syslog RFC series. + * rgerhards, 2008-10-09 + * We check if we have a NUL character at the very end of the + * message. This seems to be a frequent problem with a number of senders. + * So I have now decided to drop these NULs. However, if they are intentional, + * that may cause us some problems, e.g. with syslog-sign. On the other hand, + * current code always has problems with intentional NULs (as it needs to escape + * them to prevent problems with the C string libraries), so that does not + * really matter. Just to be on the save side, we'll log destruction of such + * NULs in the debug log. + * rgerhards, 2007-09-14 + */ +static inline rsRetVal +sanitizeMessage(msg_t *pMsg) +{ + DEFiRet; + uchar *pszMsg; + uchar *pDst; /* destination for copy job */ + size_t lenMsg; + size_t iSrc; + size_t iDst; + size_t iMaxLine; + + assert(pMsg != NULL); + +# ifdef USE_NETZIP + CHKiRet(uncompressMessage(pMsg)); +# endif + + pszMsg = pMsg->pszRawMsg; + lenMsg = pMsg->iLenRawMsg; + + /* remove NUL character at end of message (see comment in function header) */ + if(pszMsg[lenMsg-1] == '\0') { + DBGPRINTF("dropped NUL at very end of message\n"); + lenMsg--; + } + + /* then we check if we need to drop trailing LFs, which often make + * their way into syslog messages unintentionally. In order to remain + * compatible to recent IETF developments, we allow the user to + * turn on/off this handling. rgerhards, 2007-07-23 + */ + if(bDropTrailingLF && pszMsg[lenMsg-1] == '\n') { + DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); + lenMsg--; + } + + /* now copy over the message and sanitize it */ + /* TODO: can we get cheaper memory alloc? {alloca()?}*/ + iMaxLine = glbl.GetMaxLine(); + CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + iSrc = iDst = 0; + while(iSrc < lenMsg && iDst < iMaxLine) { + if(pszMsg[iSrc] == '\0') { /* guard against \0 characters... */ + /* changed to the sequence (somewhat) proposed in + * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 + */ + if(iDst + 3 < iMaxLine) { /* do we have space? */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0'; + pDst[iDst++] = '0'; + pDst[iDst++] = '0'; + } /* if we do not have space, we simply ignore the '\0'... */ + /* log an error? Very questionable... rgerhards, 2006-11-30 */ + /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ + } else if(bEscapeCCOnRcv && iscntrl((int) pszMsg[iSrc])) { + /* we are configured to escape control characters. Please note + * that this most probably break non-western character sets like + * Japanese, Korean or Chinese. rgerhards, 2007-07-17 + * Note: sysklogd logs octal values only for DEL and CCs above 127. + * For others, it logs ^n where n is the control char converted to an + * alphabet character. We like consistency and thus escape it to octal + * in all cases. If someone complains, we may change the mode. At least + * we known now what's going on. + * rgerhards, 2007-07-17 + */ + if(iDst + 3 < iMaxLine) { /* do we have space? */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); + } /* again, if we do not have space, we ignore the char - see comment at '\0' */ + } else { + pDst[iDst++] = pszMsg[iSrc]; + } + ++iSrc; + } + pDst[iDst] = '\0'; /* space *is* reserved for this! */ + + /* we have a sanitized string. Let's save it now */ + free(pMsg->pszRawMsg); + if((pMsg->pszRawMsg = malloc((iDst+1) * sizeof(uchar))) == NULL) { + /* when we get no new buffer, we use what we already have ;) */ + pMsg->pszRawMsg = pDst; + } else { + /* trim buffer */ + memcpy(pMsg->pszRawMsg, pDst, iDst+1); + free(pDst); /* too big! */ + pMsg->iLenRawMsg = iDst; + } + +finalize_it: + RETiRet; +} + +/* Parse a received message. The object's rawmsg property is taken and + * parsed according to the relevant standards. This can later be + * extended to support configured parsers. + * rgerhards, 2008-10-09 + */ +rsRetVal parseMsg(msg_t *pMsg) +{ + DEFiRet; + uchar *msg; + int pri; + + CHKiRet(sanitizeMessage(pMsg)); + + /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ + DBGPRINTF("msg parser: flags %x, from '%s', msg %s\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); + + /* pull PRI */ + pri = DEFUPRI; + msg = pMsg->pszRawMsg; + if(*msg == '<') { + pri = 0; + while(isdigit((int) *++msg)) { + pri = 10 * pri + (*msg - '0'); + } + if(*msg == '>') + ++msg; + if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) + pri = DEFUPRI; + } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetUxTradMsg(pMsg, (char*) msg); + + if(pMsg->bParseHOSTNAME == 0) + MsgSetHOSTNAME(pMsg, (char*) pMsg->pszRcvFrom); + + /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have + * a traditional syslog message or one formatted according to syslog-protocol. + * We need to apply different parsers depending on that. We use the + * -protocol VERSION field for the detection. + */ + if(msg[0] == '1' && msg[1] == ' ') { + dbgprintf("Message has syslog-protocol format.\n"); + setProtocolVersion(pMsg, 1); + if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { // TODO: parseRFC... should pull flags from pMsg + msgDestruct(&pMsg); + ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + } + } else { /* we have legacy syslog */ + dbgprintf("Message has legacy syslog format.\n"); + setProtocolVersion(pMsg, 0); + if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) { + msgDestruct(&pMsg); + ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + } + } + + /* finalize message object */ + pMsg->bIsParsed = 1; /* this message is now parsed */ + MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */ + +finalize_it: + RETiRet; +} diff --git a/runtime/parser.h b/runtime/parser.h new file mode 100644 index 00000000..cec9c083 --- /dev/null +++ b/runtime/parser.h @@ -0,0 +1,30 @@ +/* header for parser.c + * This is not yet an object, but contains all those code necessary to + * parse syslog messages. + * + * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_PARSE_H +#define INCLUDED_PARSE_H + +extern rsRetVal parserClassInit(void); +extern rsRetVal parseMsg(msg_t*); + +#endif /* #ifndef INCLUDED_PARSE_H */ diff --git a/runtime/queue.c b/runtime/queue.c index f5f770b3..25c0bd5f 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1513,8 +1513,6 @@ queueRateLimiter(queue_t *pThis) ISOBJ_TYPE_assert(pThis, queue); - dbgoprint((obj_t*) pThis, "entering rate limiter\n"); - iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ /* time calls are expensive, so only do them when needed */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 361bfb47..619343bd 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -252,6 +252,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_QUEUE_FULL = -2105, /**< queue is full, operation could not be completed */ RS_RET_ACCEPT_ERR = -2106, /**< error during accept() system call */ RS_RET_INVLD_TIME = -2107, /**< invalid timestamp (e.g. could not be parsed) */ + RS_RET_NO_ZIP = -2108, /**< ZIP functionality is not present */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/wti.c b/runtime/wti.c index 365b25d5..4abca090 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -299,7 +299,7 @@ wtiWorkerCancelCleanup(void *arg) pWtp = pThis->pWtp; ISOBJ_TYPE_assert(pWtp, wtp); - dbgprintf("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); /* call user supplied handler (that one e.g. requeues the element) */ pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp); @@ -391,7 +391,7 @@ wtiWorker(wti_t *pThis) /* if we reach this point, we are still protected by the mutex */ if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) { - dbgprintf("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); if(pWtp->toWrkShutdown == -1) { @@ -400,7 +400,7 @@ wtiWorker(wti_t *pThis) } else { timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { - dbgprintf("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); bInactivityTOOccured = 1; /* indicate we had a timeout */ } } diff --git a/tools/syslogd.c b/tools/syslogd.c index e794e2d1..1a26333d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -128,6 +128,7 @@ #include "vm.h" #include "errmsg.h" #include "datetime.h" +#include "parser.h" #include "sysvar.h" /* definitions for objects we access */ @@ -249,15 +250,15 @@ typedef struct legacyOptsLL_s { legacyOptsLL_t *pLegacyOptsLL = NULL; /* global variables for config file state */ -static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ +int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ int iCompatibilityMode = 0; /* version we should be compatible with; 0 means sysklogd. It is the default, so if no -c option is given, we make ourselvs as compatible to sysklogd as possible. */ static int bDebugPrintTemplateList = 1;/* output template list in debug mode? */ static int bDebugPrintCfSysLineHandlerList = 1;/* output cfsyslinehandler list in debug mode? */ static int bDebugPrintModuleList = 1;/* output module list in debug mode? */ -static uchar cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */ -static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +uchar cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */ +int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ static int bErrMsgToStderr = 1; /* print error messages to stderr (in addition to everything else)? */ int bReduceRepeatMsgs; /* reduce repeated message - 0 - no, 1 - yes */ int bActExecWhenPrevSusp; /* execute action only when previous one was suspended? */ @@ -596,6 +597,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b int pri; msg_t *pMsg; + pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ /* Now it is time to create the message object (rgerhards) */ if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); @@ -1190,6 +1192,9 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) assert(pMsg != NULL); + if(pMsg->bIsParsed == 0) { + parseMsg(pMsg); + } processMsg(pMsg); msgDestruct(&pMsg); @@ -1311,7 +1316,7 @@ static int parseRFCStructuredData(char **pp2parse, char *pResult) * * rger, 2005-11-24 */ -static int parseRFCSyslogMsg(msg_t *pMsg, int flags) +int parseRFCSyslogMsg(msg_t *pMsg, int flags) { char *p2parse; char *pBuf; @@ -1407,7 +1412,7 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags) * but I thought I log it in this comment. * rgerhards, 2006-01-10 */ -static int parseLegacySyslogMsg(msg_t *pMsg, int flags) +int parseLegacySyslogMsg(msg_t *pMsg, int flags) { char *p2parse; char *pBuf; @@ -2908,6 +2913,8 @@ InitGlobalClasses(void) CHKiRet(actionClassInit()); pErrObj = "template"; CHKiRet(templateInit()); + pErrObj = "parser"; + CHKiRet(parserClassInit()); /* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ pErrObj = "net"; @@ -3526,6 +3533,5 @@ int main(int argc, char **argv) dbgClassInit(); return realMain(argc, argv); } - /* vim:set ai: */ -- cgit From 5742f9cdd0da18a3ddfb0a51a981637124a6ab25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 07:48:22 +0200 Subject: fixing segfault caused by all inputs but imudp --- runtime/msg.c | 1 + tools/syslogd.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/msg.c b/runtime/msg.c index c030fa45..e52c9e3f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -264,6 +264,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) /* initialize members that are non-zero */ pM->iRefCount = 1; + pM->bIsParsed = 1; /* first we assume this is parsed. If not, input must re-set to 0 */ pM->iSeverity = -1; pM->iFacility = -1; objConstructSetObjInfo(pM); diff --git a/tools/syslogd.c b/tools/syslogd.c index 1a26333d..a45942fa 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -597,13 +597,13 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b int pri; msg_t *pMsg; - pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ /* Now it is time to create the message object (rgerhards) */ if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } + pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); @@ -1192,6 +1192,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) assert(pMsg != NULL); +RUNLOG_VAR("%d", pMsg->bIsParsed); if(pMsg->bIsParsed == 0) { parseMsg(pMsg); } -- cgit From 39f2afc8b8a60f09570fcf288e6899cdd42209ba Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 09:16:31 +0200 Subject: bumped interface version number to reflect change to message parsing --- configure.ac | 2 +- runtime/modules.h | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 6b6a699d..e1bc248c 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[3.21.5],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[3.21.6-Test1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/runtime/modules.h b/runtime/modules.h index 7d34bcf7..96016082 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -44,8 +44,11 @@ * rgerhards, 2008-03-04 * version 3 adds modInfo_t ptr to call of modInit -- rgerhards, 2008-03-10 * version 4 removes needUDPSocket OM callback -- rgerhards, 2008-03-22 + * version 5 changes the way parsing works for input modules. This is + * an important change, parseAndSubmitMessage() goes away. Other + * module types are not affected. -- rgerhards, 2008-10-09 */ -#define CURR_MOD_IF_VERSION 4 +#define CURR_MOD_IF_VERSION 5 typedef enum eModType_ { eMOD_IN, /* input module */ -- cgit From 634cfdbe8ec7182b7fb74b184d269697e5fcab6c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 10:01:30 +0200 Subject: restoring msg parsing for imudp I tried to work too quick this morning. A side-effect of an earlier change was that no UDP messages were parsed, which lead to their loss, because no PRI was set in this case. --- plugins/imudp/imudp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index aab201a9..114b433c 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -207,6 +207,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); + pMsg->bIsParsed = 0; /* indicate message needs to be parsed */ pMsg->iLenRawMsg = lenRcvBuf; MsgSetInputName(pMsg, "imudp"); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); -- cgit From af3c944563b8651c14b2b9087ea76f7eeacfc065 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 11:21:24 +0200 Subject: added experimental pthread_yield() which so far seems to increase performance. There is also reason for it to do so, see http://kb.monitorware.com/post14216.html#p14216 --- runtime/queue.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/queue.c b/runtime/queue.c index 25c0bd5f..3ef6d5a0 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2185,6 +2185,11 @@ finalize_it: /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); + /* the following pthread_yield is experimental, but brought us performance + * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 + * rgerhards, 2008-10-09 + */ + pthread_yield(); } RETiRet; -- cgit From 1229143ca3d194bb0daaa5252809d59ee0c3a0bf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 12:56:28 +0200 Subject: minor: reorder to slightly reduce size of critical section --- runtime/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/queue.c b/runtime/queue.c index 3ef6d5a0..2d1bf7e3 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2181,10 +2181,10 @@ finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { /* make sure at least one worker is running. */ queueAdviseMaxWorkers(pThis); - dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); + dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); /* the following pthread_yield is experimental, but brought us performance * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 * rgerhards, 2008-10-09 -- cgit From 6c6e9a0f3f7d454ba9553a750b195d7f99c7299a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 13:45:56 +0200 Subject: moved bParseHostname and bIsParsed to msgFlags This enables us to use more efficient calling conventions and also helps us keep the on-disk structure of a msg object more consistent in future releases. --- dirty.h | 13 +------------ plugins/immark/immark.c | 1 + plugins/imrelp/imrelp.c | 5 +++-- plugins/imudp/imudp.c | 4 +--- plugins/imuxsock/imuxsock.c | 4 +++- runtime/atomic.h | 2 ++ runtime/debug.h | 1 + runtime/msg.c | 1 - runtime/msg.h | 13 ++++++++++++- runtime/parser.c | 3 +-- runtime/queue.c | 5 +++-- tcps_sess.c | 11 ++++++----- tools/syslogd.c | 21 ++++++++++++--------- 13 files changed, 46 insertions(+), 38 deletions(-) diff --git a/dirty.h b/dirty.h index f5d0415e..4b13adcd 100644 --- a/dirty.h +++ b/dirty.h @@ -27,20 +27,9 @@ #ifndef DIRTY_H_INCLUDED #define DIRTY_H_INCLUDED 1 -/* Flags to logmsg(). - */ -#define NOFLAG 0x000 /* no flag is set (to be used when a flag must be specified and none is required) */ -#define INTERNAL_MSG 0x001 /* msg generated by logmsgInternal() --> special handling */ -/* NO LONGER USED: #define SYNC_FILE 0x002 / * do fsync on file after printing */ -#define IGNDATE 0x004 /* ignore, if given, date in message and use date of reception as msg date */ -#define MARK 0x008 /* this message is a mark */ - -#define MSG_PARSE_HOSTNAME 1 -#define MSG_DONT_PARSE_HOSTNAME 0 - rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); -rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); +rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/plugins/immark/immark.c b/plugins/immark/immark.c index 323da3fe..8504f872 100644 --- a/plugins/immark/immark.c +++ b/plugins/immark/immark.c @@ -41,6 +41,7 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "msg.h" MODULE_TYPE_INPUT diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 4515acd7..2255e643 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -42,6 +42,7 @@ #include "cfsysline.h" #include "module-template.h" #include "net.h" +#include "msg.h" MODULE_TYPE_INPUT @@ -83,8 +84,8 @@ static relpRetVal onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, size_t lenMsg) { DEFiRet; - parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); + parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, PARSE_HOSTNAME, + eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); RETiRet; } diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 114b433c..a49378cf 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -207,12 +207,10 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); - pMsg->bIsParsed = 0; /* indicate message needs to be parsed */ pMsg->iLenRawMsg = lenRcvBuf; MsgSetInputName(pMsg, "imudp"); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - pMsg->bParseHOSTNAME = MSG_PARSE_HOSTNAME; - pMsg->msgFlags = NOFLAG; + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; MsgSetRcvFrom(pMsg, (char*)fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); CHKiRet(submitMsg(pMsg)); diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index efa0365d..1d88a2b5 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -42,6 +42,7 @@ #include "errmsg.h" #include "net.h" #include "glbl.h" +#include "msg.h" MODULE_TYPE_INPUT @@ -221,7 +222,8 @@ static rsRetVal readSocket(int fd, int iSock) if (iRcvd > 0) { parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); + iRcvd, funixParseHost[iSock] ? (funixFlags[iSock] | PARSE_HOSTNAME) : funixFlags[iSock], + funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); diff --git a/runtime/atomic.h b/runtime/atomic.h index 2dbe7f52..7ad8e2e4 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -42,12 +42,14 @@ */ #ifdef HAVE_ATOMIC_BUILTINS # define ATOMIC_INC(data) ((void) __sync_fetch_and_add(&(data), 1)) +# define ATOMIC_DEC(data) ((void) __sync_sub_and_fetch(&(data), 1)) # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) #else # warning "atomic builtins not available, using nul operations" # define ATOMIC_INC(data) (++(data)) +# define ATOMIC_DEC(data) (--(data)) # define ATOMIC_DEC_AND_FETCH(data) (--(data)) # define ATOMIC_FETCH_32BIT(data) (data) # define ATOMIC_STORE_1_TO_32BIT(data) (data) = 1 diff --git a/runtime/debug.h b/runtime/debug.h index 7ac29765..59b3e776 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -101,6 +101,7 @@ void dbgPrintAllDebugInfo(void); /* macros */ #define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } +#define DBGOPRINT(...) if(Debug) { dbgoprint(__VA_ARGS__); } #ifdef RTINST # define BEGINfunc static dbgFuncDB_t *pdbgFuncDB; int dbgCALLStaCK_POP_POINT = dbgEntrFunc(&pdbgFuncDB, __FILE__, __func__, __LINE__); # define ENDfunc dbgExitFunc(pdbgFuncDB, dbgCALLStaCK_POP_POINT, RS_RET_NO_IRET); diff --git a/runtime/msg.c b/runtime/msg.c index e52c9e3f..c030fa45 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -264,7 +264,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) /* initialize members that are non-zero */ pM->iRefCount = 1; - pM->bIsParsed = 1; /* first we assume this is parsed. If not, input must re-set to 0 */ pM->iSeverity = -1; pM->iFacility = -1; objConstructSetObjInfo(pM); diff --git a/runtime/msg.h b/runtime/msg.h index 98635f85..d98111a8 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -55,7 +55,6 @@ struct msg { flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ - short bIsParsed; /* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */ short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -122,6 +121,18 @@ struct msg { int msgFlags; /* flags associated with this message */ }; + +/* message flags (msgFlags), not an enum for historical reasons + */ +#define NOFLAG 0x000 /* no flag is set (to be used when a flag must be specified and none is required) */ +#define INTERNAL_MSG 0x001 /* msg generated by logmsgInternal() --> special handling */ +/* 0x002 not used because it was previously a known value - rgerhards, 2008-10-09 */ +#define IGNDATE 0x004 /* ignore, if given, date in message and use date of reception as msg date */ +#define MARK 0x008 /* this message is a mark */ +#define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ +#define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ + + /* function prototypes */ PROTOTYPEObjClassInit(msg); diff --git a/runtime/parser.c b/runtime/parser.c index 8c4272a0..fbdeebeb 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -41,7 +41,6 @@ /* some defines */ #define DEFUPRI (LOG_USER|LOG_NOTICE) -#warning "msg object must be updated with new property for persisting the queue!" /* definitions for objects we access */ DEFobjStaticHelpers DEFobjCurrIf(glbl) @@ -306,7 +305,7 @@ rsRetVal parseMsg(msg_t *pMsg) } /* finalize message object */ - pMsg->bIsParsed = 1; /* this message is now parsed */ + pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */ MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */ finalize_it: diff --git a/runtime/queue.c b/runtime/queue.c index b0043ef5..42b8137d 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -49,6 +49,7 @@ #include "obj.h" #include "wtp.h" #include "wti.h" +#include "atomic.h" /* static data */ DEFobjStaticHelpers @@ -996,7 +997,7 @@ queueAdd(queue_t *pThis, void *pUsr) CHKiRet(pThis->qAdd(pThis, pUsr)); if(pThis->qType != QUEUETYPE_DIRECT) { - ++pThis->iQueueSize; + ATOMIC_INC(pThis->iQueueSize); dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize); } @@ -1025,7 +1026,7 @@ queueDel(queue_t *pThis, void *pUsr) iRet = queueGetUngottenObj(pThis, (obj_t**) pUsr); } else { iRet = pThis->qDel(pThis, pUsr); - --pThis->iQueueSize; + ATOMIC_DEC(pThis->iQueueSize); } dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n", diff --git a/tcps_sess.c b/tcps_sess.c index 13644f45..e8bef5b1 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -42,6 +42,7 @@ #include "obj.h" #include "errmsg.h" #include "netstrm.h" +#include "msg.h" /* static data */ @@ -229,8 +230,8 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->bAtStrtOfFram = 1; } @@ -313,7 +314,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -324,7 +325,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -343,7 +344,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tools/syslogd.c b/tools/syslogd.c index a45942fa..13696955 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -588,8 +588,11 @@ void untty(void) * a timestamp that is to be used as timegenerated instead of the current system time. * This is meant to facilitate performance optimization. Some inputs support such modes. * If stTime is NULL, the current system time is used. + * + * rgerhards, 2008-10-09: + * interface change: bParseHostname removed, now in flags */ -static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, +static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int flags, flowControl_t flowCtlType, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; @@ -603,13 +606,11 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b } else { CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } - pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsg(pMsg, (char*)msg); - pMsg->bParseHOSTNAME = bParseHost; /* test for special codes */ pri = DEFUPRI; p = msg; @@ -634,7 +635,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b * the message was received from (that, for obvious reasons, * being the local host). rgerhards 2004-11-16 */ - if(bParseHost == 0) + if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) MsgSetHOSTNAME(pMsg, (char*)hname); MsgSetRcvFrom(pMsg, (char*)hname); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); @@ -702,9 +703,12 @@ finalize_it: * a timestamp that is to be used as timegenerated instead of the current system time. * This is meant to facilitate performance optimization. Some inputs support such modes. * If stTime is NULL, the current system time is used. + * + * rgerhards, 2008-10-09: + * interface change: bParseHostname removed, now in flags */ rsRetVal -parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType, +parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; @@ -816,7 +820,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); + printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); } else { /* This case in theory never can happen. If it happens, we have * a logic error. I am checking for it, because if I would not, @@ -868,7 +872,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); + printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); finalize_it: if(tmpline != NULL) @@ -1192,8 +1196,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) assert(pMsg != NULL); -RUNLOG_VAR("%d", pMsg->bIsParsed); - if(pMsg->bIsParsed == 0) { + if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); } processMsg(pMsg); -- cgit From a27e249e445deecb1d9ef57fbbb203d71bf061dd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Oct 2008 10:55:33 +0200 Subject: bugfix: (potentially big) memory leak on HUP This occured if queues could not be drained before timeout. Thanks to David Lang for pointing this out. --- ChangeLog | 2 ++ runtime/queue.c | 45 ++++++++++++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2a3c9acf..9a02d675 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,8 @@ version before switching to this one. and consistency - bugfix: subsecond time properties generated by imfile, imklog and internal messages could be slightly inconsistent +- bugfix: (potentially big) memory leak on HUP if queues could not be + drained before timeout - thanks to David Lang for pointing this out --------------------------------------------------------------------------- Version 3.21.5 [DEVEL] (rgerhards), 2008-09-30 - performance optimization: unnecessary time() calls during message diff --git a/runtime/queue.c b/runtime/queue.c index 42b8137d..7fa2df91 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -88,6 +88,30 @@ ENDfunc return pThis->iQueueSize + pThis->iUngottenObjs; } + +/* This function drains the queue in cases where this needs to be done. The most probable + * reason is a HUP which needs to discard data (because the queue is configured to be lossy). + * During a shutdown, this is typically not needed, as the OS frees up ressources and does + * this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21 + * This function returns void, as it makes no sense to communicate an error back, even if + * it happens. + */ +static inline void queueDrain(queue_t *pThis) +{ + void *pUsr; + + ASSERT(pThis != NULL); + + /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ + while(pThis->iQueueSize-- > 0) { + pThis->qDel(pThis, &pUsr); + if(pUsr != NULL) { + objDestruct(pUsr); + } + } +} + + /* --------------- code for disk-assisted (DA) queue modes -------------------- */ @@ -196,14 +220,6 @@ queueTurnOffDAMode(queue_t *pThis) } } - /* TODO: we have a *really biiiiig* memory leak here: if the queue could not be persisted, all of - * its data elements are still in memory. That doesn't really matter if we are terminated, but on - * HUP this memory leaks. We MUST add a loop of destructor calls here. However, this takes time - * (possibly a lot), so it is probably best to have a config variable for that. - * Something for 3.11.1! - * rgerhards, 2008-01-30 - */ - RETiRet; } @@ -461,12 +477,15 @@ static rsRetVal qDestructFixedArray(queue_t *pThis) ASSERT(pThis != NULL); + queueDrain(pThis); /* discard any remaining queue entries */ + if(pThis->tVars.farray.pBuf != NULL) free(pThis->tVars.farray.pBuf); RETiRet; } + static rsRetVal qAddFixedArray(queue_t *pThis, void* in) { DEFiRet; @@ -570,11 +589,11 @@ static rsRetVal qConstructLinkedList(queue_t *pThis) static rsRetVal qDestructLinkedList(queue_t __attribute__((unused)) *pThis) { DEFiRet; - - /* with the linked list type, there is nothing to do here. The - * reason is that the Destructor is only called after all entries - * have bene taken off the queue. In this case, there is nothing - * dynamic left with the linked list. + + queueDrain(pThis); /* discard any remaining queue entries */ + + /* with the linked list type, there is nothing left to do here. The + * reason is that there are no dynamic elements for the list itself. */ RETiRet; -- cgit From cf38fc81759b01af5125b1a05e0d6fe8e2e1bc21 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Oct 2008 13:54:40 +0200 Subject: added a setting "$OptimizeForUniprocessor" ...to enable users to turn off pthread_yield calls which are counter-productive on multiprocessor machines (but have been shown to be useful on uniprocessors) --- ChangeLog | 3 +++ doc/rsyslog_conf.html | 6 ++++-- runtime/glbl.c | 5 +++++ runtime/glbl.h | 1 + runtime/queue.c | 5 ++++- runtime/queue.h | 1 + runtime/wti.c | 15 ++++++++++++++- runtime/wti.h | 1 + runtime/wtp.c | 16 +++++++++++++++- runtime/wtp.h | 1 + 10 files changed, 49 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 41aef49f..6856c3e7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,9 @@ version before switching to this one. receive loop (aka receiving messsages at a high rate) - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period +- added a setting "$OptimizeForUniprocessor" to enable users to turn off + pthread_yield calls which are counter-productive on multiprocessor + machines (but have been shown to be useful on uniprocessors) --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - consolidated time calls during msg object creation, improves performance diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 2b773476..be543833 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -236,8 +236,10 @@ supported in order to be compliant to the upcoming new syslog RFC series.

  • $ModLoad
  • $RepeatedMsgReduction
  • $ResetConfigVariables
    • -
  • $WorkDirectory <name> (directory for spool -and other work files)
    • +
  • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better +performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The +default may change as uniprocessor systems become less common.
    • +
  • $WorkDirectory <name> (directory for spool and other work files)
  • $UDPServerAddress <IP> (imudp) -- local IP address (or name) the UDP listens should bind to
  • $UDPServerRun <port> (imudp) -- former diff --git a/runtime/glbl.c b/runtime/glbl.c index 1114fcd3..c752288f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -51,6 +51,7 @@ DEFobjStaticHelpers * class... */ static uchar *pszWorkDir = NULL; +static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ @@ -85,6 +86,7 @@ static dataType Get##nameFunc(void) \ return(nameVar); \ } +SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) @@ -173,6 +175,7 @@ CODESTARTobjQueryInterface(glbl) pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; SIMP_PROP(MaxLine); + SIMP_PROP(OptimizeUniProc); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); @@ -216,6 +219,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a pszWorkDir = NULL; } bDropMalPTRMsgs = 0; + bOptimizeUniProc = 1; return RS_RET_OK; } @@ -235,6 +239,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) diff --git a/runtime/glbl.h b/runtime/glbl.h index 0c83bdd5..0b871b3c 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -41,6 +41,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ dataType (*Get##name)(void); \ rsRetVal (*Set##name)(dataType); SIMP_PROP(MaxLine, int) + SIMP_PROP(OptimizeUniProc, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) diff --git a/runtime/queue.c b/runtime/queue.c index 7fa2df91..a3e29a96 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1285,6 +1285,7 @@ rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); + pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -2208,8 +2209,10 @@ finalize_it: /* the following pthread_yield is experimental, but brought us performance * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 * rgerhards, 2008-10-09 + * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 */ - pthread_yield(); + if(pThis->bOptimizeUniProc) + pthread_yield(); } RETiRet; diff --git a/runtime/queue.h b/runtime/queue.h index 9e75b31b..a2dd594f 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -58,6 +58,7 @@ typedef struct qWrkThrd_s { typedef struct queue_s { BEGINobjInstance; queueType_t qType; + int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ int bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ int bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ int bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ diff --git a/runtime/wti.c b/runtime/wti.c index 4abca090..e8a77474 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -45,9 +45,11 @@ #include "wtp.h" #include "wti.h" #include "obj.h" +#include "glbl.h" /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) /* forward-definitions */ @@ -202,6 +204,7 @@ ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ + pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_cond_init(&pThis->condExitDone, NULL); pthread_mutex_init(&pThis->mut, NULL); ENDobjConstruct(wti) @@ -367,7 +370,8 @@ wtiWorker(wti_t *pThis) wtpProcessThrdChanges(pWtp); pthread_testcancel(); /* see big comment in function header */ # if !defined(__hpux) /* pthread_yield is missing there! */ - pthread_yield(); /* see big comment in function header */ + if(pThis->bOptimizeUniProc) + pthread_yield(); /* see big comment in function header */ # endif /* if we have a rate-limiter set for this worker pool, let's call it. Please @@ -466,6 +470,14 @@ finalize_it: /* dummy */ rsRetVal wtiQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } +/* exit our class + */ +BEGINObjClassExit(wti, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdsel_gtls) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(wti) + /* Initialize the wti class. Must be called as the very first method * before anything else is called inside this class. @@ -473,6 +485,7 @@ rsRetVal wtiQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } */ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most important for persisting) */ /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wti) /* diff --git a/runtime/wti.h b/runtime/wti.h index 0cd6744e..6b60b833 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -31,6 +31,7 @@ /* the worker thread instance class */ typedef struct wti_s { BEGINobjInstance; + int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ pthread_t thrdID; /* thread ID */ qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 45034fa7..06173e04 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -46,9 +46,11 @@ #include "wtp.h" #include "wti.h" #include "obj.h" +#include "glbl.h" /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) /* forward-definitions */ @@ -75,6 +77,7 @@ static rsRetVal NotImplementedDummy() { return RS_RET_OK; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ + pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_mutex_init(&pThis->mut, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); /* set all function pointers to "not implemented" dummy so that we can safely call them */ @@ -484,7 +487,8 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) * hold the queue's mutex, but at least it has a chance to start on a single-CPU system. */ # if !defined(__hpux) /* pthread_yield is missing there! */ - pthread_yield(); + if(pThis->bOptimizeUniProc) + pthread_yield(); # endif /* indicate we just started a worker and would like to see it running */ @@ -617,12 +621,22 @@ finalize_it: /* dummy */ rsRetVal wtpQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } +/* exit our class + */ +BEGINObjClassExit(wtp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdsel_gtls) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(wtp) + + /* Initialize the stream class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-01-09 */ BEGINObjClassInit(wtp, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wtp) /* diff --git a/runtime/wtp.h b/runtime/wtp.h index 13ebe536..88d88afd 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -52,6 +52,7 @@ typedef enum { /* the worker thread pool (wtp) object */ typedef struct wtp_s { BEGINobjInstance; + int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ wtpState_t wtpState; int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ -- cgit From 6334d335d89ae5df344f833c5095e9dea2abf6fb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Oct 2008 14:46:47 +0200 Subject: added configuration directive "HUPisRestart" ...which enables to configure HUP to be either a full restart or "just" a leightweight way to close open files --- ChangeLog | 12 ++++-- action.c | 33 +++++++++++++++ action.h | 1 + configure.ac | 2 +- doc/manual.html | 2 +- doc/rsyslog_conf.html | 20 +++++---- runtime/glbl.c | 5 +++ runtime/glbl.h | 1 + runtime/module-template.h | 31 +++++++++++++- runtime/modules.c | 5 +++ runtime/modules.h | 1 + tools/omfile.c | 102 +++++++++++++++++++++++++++++----------------- tools/syslogd.c | 59 ++++++++++++++++++++++----- 13 files changed, 212 insertions(+), 62 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6856c3e7..a11ce27d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 3.23.1 [DEVEL] (rgerhards), 2008-10-?? +Version 4.1.0 [DEVEL] (rgerhards), 2008-10-?? ********************************* WARNING ********************************* This version has a slightly different on-disk format for message entries. @@ -9,6 +9,13 @@ the output driver. It is recommended to drain queues with the previous version before switching to this one. ********************************* WARNING ********************************* +- greatly enhanced performance when compared to v3. +- added configuration directive "HUPisRestart" which enables to configure + HUP to be either a full restart or "just" a leightweight way to + close open files. +- added a setting "$OptimizeForUniprocessor" to enable users to turn off + pthread_yield calls which are counter-productive on multiprocessor + machines (but have been shown to be useful on uniprocessors) - reordered imudp processing. Message parsing is now done as part of main message queue worker processing (was part of the input thread) This should also improve performance, as potentially more work is @@ -21,9 +28,6 @@ version before switching to this one. receive loop (aka receiving messsages at a high rate) - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period -- added a setting "$OptimizeForUniprocessor" to enable users to turn off - pthread_yield calls which are counter-productive on multiprocessor - machines (but have been shown to be useful on uniprocessors) --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - consolidated time calls during msg object creation, improves performance diff --git a/action.c b/action.c index ec8732bc..dee46f16 100644 --- a/action.c +++ b/action.c @@ -498,6 +498,39 @@ finalize_it: } #pragma GCC diagnostic warning "-Wempty-body" + +/* call the HUP handler for a given action, if such a handler is defined. The + * action mutex is locked, because the HUP handler most probably needs to modify + * some internal state information. + * rgerhards, 2008-10-22 + */ +#pragma GCC diagnostic ignored "-Wempty-body" +rsRetVal +actionCallHUPHdlr(action_t *pAction) +{ + DEFiRet; + int iCancelStateSave; + + ASSERT(pAction != NULL); + DBGPRINTF("Action %p checks HUP hdlr: %p\n", pAction, pAction->pMod->doHUP); + + if(pAction->pMod->doHUP == NULL) { + FINALIZE; /* no HUP handler, so we are done ;) */ + } + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(&pAction->mutActExec); + pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); + pthread_setcancelstate(iCancelStateSave, NULL); + CHKiRet(pAction->pMod->doHUP(pAction->pModData)); + pthread_cleanup_pop(1); /* unlock mutex */ + +finalize_it: + RETiRet; +} +#pragma GCC diagnostic warning "-Wempty-body" + + /* set the action message queue mode * TODO: probably move this into queue object, merge with MainMsgQueue! * rgerhards, 2008-01-28 diff --git a/action.h b/action.h index d26d15b6..8d9d5102 100644 --- a/action.h +++ b/action.h @@ -85,6 +85,7 @@ rsRetVal actionSetGlobalResumeInterval(int iNewVal); rsRetVal actionDoAction(action_t *pAction); rsRetVal actionCallAction(action_t *pAction, msg_t *pMsg); rsRetVal actionWriteToAction(action_t *pAction); +rsRetVal actionCallHUPHdlr(action_t *pAction); rsRetVal actionClassInit(void); rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, int bSuspended); diff --git a/configure.ac b/configure.ac index 2f17e575..981ef193 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[3.23.1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/manual.html b/doc/manual.html index 884c43c0..f1b7d657 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,7 +16,7 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

    -

    This documentation is for version 3.21.6 (devel branch) of rsyslog. +

    This documentation is for version 4.1.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index be543833..cd3db405 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -59,17 +59,19 @@ remember, that a modules configuration directive (and functionality) is only available if it has been loaded (using $ModLoad).

    Lines

    Lines can be continued by specifying a backslash ("\") as the last -character of the line.
    +character of the line. There is a hard-coded maximum line length of 4K. +If you need lines larger than that, you need to change compile-time +settings inside rsyslog and recompile.

    Global Directives

    All global directives need to be specified on a line by their own and must start with a dollar-sign. Here is a list in alphabetical order. Follow links for a description.

    +

    Please note that not all directives here are actually global. Some affect +only the next action. This documentation will be changed soon.

    Not all directives have an in-depth description right now. Default values for them are in bold. A more in-depth description will -appear as implementation progresses. Directives may change during that -process, we will NOT try hard to maintain backwards compatibility -(after all, v3 is still very early in development and quite -unstable...). So you have been warned ;)

    +appear as implementation progresses. +

    Be sure to read information about queues in rsyslog - many parameter settings modify queue parameters. If in doubt, use the default, it is usually well-chosen and applicable in most cases.

    @@ -175,12 +177,16 @@ default 60000 (1 minute)]
  • $GssForwardServiceName
  • $GssListenServiceName
  • $GssMode
    • +
  • $HUPisRestart [on/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending +on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is +not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for +compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually +the only one that is needed e.g. for log rotation. It is recommended to set the setting to "off".
  • $IncludeConfig
  • MainMsgQueueCheckpointInterval <number>
  • $MainMsgQueueDequeueSlowdown <number> [number is timeout in microseconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]
    • -
  • $MainMsgQueueDiscardMark <number> [default -9750]
    • +
  • $MainMsgQueueDiscardMark <number> [default 9750]
  • $MainMsgQueueDiscardSeverity <severity> [either a textual or numerical severity! default 4 (warning)]
  • $MainMsgQueueFileName <name>
    • diff --git a/runtime/glbl.c b/runtime/glbl.c index c752288f..2a6bfb11 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -52,6 +52,7 @@ DEFobjStaticHelpers */ static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ +static int bHUPisRestart = 1; /* should SIGHUP cause a full system restart? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ @@ -87,6 +88,7 @@ static dataType Get##nameFunc(void) \ } SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) +SIMP_PROP(HUPisRestart, bHUPisRestart, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) @@ -176,6 +178,7 @@ CODESTARTobjQueryInterface(glbl) pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(HUPisRestart); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); @@ -220,6 +223,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; + bHUPisRestart = 1; return RS_RET_OK; } @@ -240,6 +244,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) diff --git a/runtime/glbl.h b/runtime/glbl.h index 0b871b3c..44e41e3e 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -42,6 +42,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Set##name)(dataType); SIMP_PROP(MaxLine, int) SIMP_PROP(OptimizeUniProc, int) + SIMP_PROP(HUPisRestart, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) diff --git a/runtime/module-template.h b/runtime/module-template.h index eb39b587..6f7d877c 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -481,6 +481,33 @@ static rsRetVal afterRun(void)\ } -/* - * vi:set ai: +/* doHUP() + * This function is optional. Currently, it is available to output plugins + * only, but may be made available to other types of plugins in the future. + * A plugin does not need to define this entry point. If if does, it gets + * called when a non-restart type of HUP is done. A plugin should register + * this function so that it can close files, connection or other ressources + * on HUP - if it can be assume the user wanted to do this as a part of HUP + * processing. Note that the name "HUP" has historical reasons, it stems back + * to the infamous SIGHUP which was sent to restart a syslogd. We still retain + * that legacy, but may move this to a different signal. + * rgerhards, 2008-10-22 + */ +#define CODEqueryEtryPt_doHUP \ + else if(!strcmp((char*) name, "doHUP")) {\ + *pEtryPoint = doHUP;\ + } +#define BEGINdoHUP \ +static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTdoHUP + +#define ENDdoHUP \ + RETiRet;\ +} + + +/* vim:set ai: */ diff --git a/runtime/modules.c b/runtime/modules.c index d5730ede..169d234b 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -347,6 +347,7 @@ static rsRetVal doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr) { DEFiRet; + rsRetVal localRet; modInfo_t *pNew = NULL; rsRetVal (*modGetType)(eModType_t *pType); @@ -391,6 +392,10 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume)); + /* try load optional interfaces */ + localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); + if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + ABORT_FINALIZE(localRet); break; case eMOD_LIB: break; diff --git a/runtime/modules.h b/runtime/modules.h index 96016082..372529ee 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -91,6 +91,7 @@ typedef struct modInfo_s { rsRetVal (*tryResume)(void*);/* called to see if module actin can be resumed now */ rsRetVal (*modExit)(void); /* called before termination or module unload */ rsRetVal (*modGetID)(void **); /* get its unique ID from module */ + rsRetVal (*doHUP)(void *); /* non-restart type HUP handler */ /* below: parse a configuration line - return if processed * or not. If not, must be parsed to next module. */ diff --git a/tools/omfile.c b/tools/omfile.c index 8144386f..00a82933 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -174,7 +174,7 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal) } iDynaFileCacheSize = iNewVal; - dbgprintf("DynaFileCacheSize changed to %d.\n", iNewVal); + DBGPRINTF("DynaFileCacheSize changed to %d.\n", iNewVal); RETiRet; } @@ -244,7 +244,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR */ pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit; -RUNLOG_VAR("%p", pszTplName); iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -327,7 +326,7 @@ static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int b if(pCache[iEntry] == NULL) FINALIZE; - dbgprintf("Removed entry %d for file '%s' from dynaCache.\n", iEntry, + DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName); /* if the name is NULL, this is an improperly initilized entry which * needs to be discarded. In this case, neither the file is to be closed @@ -349,9 +348,11 @@ finalize_it: } -/* This function frees the dynamic file name cache. +/* This function frees all dynamic file name cache entries and closes the + * relevant files. Part of Shutdown and HUP processing. + * rgerhards, 2008-10-23 */ -static void dynaFileFreeCache(instanceData *pData) +static inline void dynaFileFreeCacheEntries(instanceData *pData) { register int i; ASSERT(pData != NULL); @@ -360,17 +361,36 @@ static void dynaFileFreeCache(instanceData *pData) for(i = 0 ; i < pData->iCurrCacheSize ; ++i) { dynaFileDelCacheEntry(pData->dynCache, i, 1); } + ENDfunc; +} + + +/* This function frees the dynamic file name cache. + */ +static void dynaFileFreeCache(instanceData *pData) +{ + ASSERT(pData != NULL); + BEGINfunc; + dynaFileFreeCacheEntries(pData); if(pData->dynCache != NULL) d_free(pData->dynCache); ENDfunc; } -/* This is a shared code for both static and dynamic files. +/* This is now shared code for all types of files. It simply prepares + * file access, which, among others, means the the file wil be opened + * and any directories in between will be created (based on config, of + * course). -- rgerhards, 2008-10-22 */ static void prepareFile(instanceData *pData, uchar *newFileName) { + if(pData->fileType == eTypePIPE) { + pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); + FINALIZE; /* we are done in this case */ + } + if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, @@ -392,8 +412,7 @@ static void prepareFile(instanceData *pData, uchar *newFileName) /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, - pData->fileGID) != 0) { + if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; close(pData->fd); @@ -409,6 +428,12 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } } } +finalize_it: + if((pData->fd) != 0 && isatty(pData->fd)) { + DBGPRINTF("file %d is a tty file\n", pData->fd); + pData->fileType = eTypeTTY; + untty(); + } } @@ -482,7 +507,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg /* we need to allocate memory for the cache structure */ pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)); if(pCache[iFirstFree] == NULL) { - dbgprintf("prepareDynfile(): could not alloc mem, discarding this request\n"); + DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n"); return -1; } } @@ -496,10 +521,11 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg * message. Otherwise, we could run into a never-ending loop. The bad * news is that we also lose errors on startup messages, but so it is. */ - if(iMsgOpts & INTERNAL_MSG) - dbgprintf("Could not open dynaFile, discarding message\n"); - else + if(iMsgOpts & INTERNAL_MSG) { + DBGPRINTF("Could not open dynaFile, discarding message\n"); + } else { errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName); + } dynaFileDelCacheEntry(pCache, iFirstFree, 1); pData->iCurrElt = -1; return -1; @@ -509,8 +535,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */ pCache[iFirstFree]->lastUsed = time(NULL); pData->iCurrElt = iFirstFree; - dbgprintf("Added new entry %d for file cache, file '%s'.\n", - iFirstFree, newFileName); + DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); return 0; } @@ -533,6 +558,8 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) ABORT_FINALIZE(RS_RET_ERR); + } else if(pData->fd == -1) { + prepareFile(pData, pData->f_fname); } /* create the message based on format specified */ @@ -584,11 +611,10 @@ again: ABORT_FINALIZE(RS_RET_OK); (void) close(pData->fd); - /* - * Check for EBADF on TTY's due to vhangup() + /* Check for EBADF on TTY's due to vhangup() * Linux uses EIO instead (mrn 12 May 96) */ - if ((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) + if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) #ifdef linux && e == EIO) { #else @@ -637,13 +663,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - dbgprintf(" (%s)\n", pData->f_fname); - /* pData->fd == -1 is an indicator that the we couldn't - * open the file at startup. For dynaFiles, this is ok, - * all others are doomed. - */ - if(pData->bDynamicName || (pData->fd != -1)) - iRet = writeFile(ppString, iMsgOpts, pData); + DBGPRINTF(" (%s)\n", pData->f_fname); + iRet = writeFile(ppString, iMsgOpts, pData); ENDdoAction @@ -726,7 +747,7 @@ CODESTARTparseSelectorAct if((pData->dynCache = (dynaFileCacheEntry**) calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) { iRet = RS_RET_OUT_OF_MEMORY; - dbgprintf("Could not allocate memory for dynaFileCache - selector disabled.\n"); + DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n"); } break; @@ -760,23 +781,15 @@ CODESTARTparseSelectorAct pData->dirUID = dirUID; pData->dirGID = dirGID; - if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); - } else { - prepareFile(pData, pData->f_fname); - } + prepareFile(pData, pData->f_fname); - if ( pData->fd < 0 ){ + if(pData->fd < 0 ) { pData->fd = -1; - dbgprintf("Error opening log file: %s\n", pData->f_fname); + DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); break; } - if (isatty(pData->fd)) { - pData->fileType = eTypeTTY; - untty(); - } - if (strcmp((char*) p, _PATH_CONSOLE) == 0) + if(strcmp((char*) p, _PATH_CONSOLE) == 0) pData->fileType = eTypeCONSOLE; break; default: @@ -811,6 +824,20 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } +BEGINdoHUP +CODESTARTdoHUP + if(pData->bDynamicName) { + dynaFileFreeCacheEntries(pData); + pData->iCurrElt = -1; /* invalidate current element */ + } else { + if(pData->fd != -1) { + close(pData->fd); + pData->fd = -1; + } + } +ENDdoHUP + + BEGINmodExit CODESTARTmodExit if(pszTplName != NULL) @@ -821,6 +848,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_doHUP ENDqueryEtryPt diff --git a/tools/syslogd.c b/tools/syslogd.c index 13696955..7145779d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -220,7 +220,7 @@ static char *PidFile = _PATH_LOGPID; /* read-only after startup */ static pid_t myPid; /* our pid for use in self-generated messages, e.g. on startup */ /* mypid is read-only after the initial fork() */ -static int restart = 0; /* do restart (config read) - multithread safe */ +static int bHadHUP = 0; /* did we have a HUP? */ static int bParseHOSTNAMEandTAG = 1; /* global config var: should the hostname and tag be * parsed inside message - rgerhards, 2006-03-13 */ @@ -2543,20 +2543,18 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz * The following function is resposible for handling a SIGHUP signal. Since * we are now doing mallocs/free as part of init we had better not being * doing this during a signal handler. Instead this function simply sets - * a flag variable which will tell the main loop to go through a restart. + * a flag variable which will tells the main loop to do "the right thing". */ void sighup_handler() { struct sigaction sigAct; - restart = 1; + bHadHUP = 1; memset(&sigAct, 0, sizeof (sigAct)); sigemptyset(&sigAct.sa_mask); sigAct.sa_handler = sighup_handler; sigaction(SIGHUP, &sigAct, NULL); - - return; } @@ -2578,6 +2576,49 @@ static void processImInternal(void) } +/* helper to doHUP(), this "HUPs" each action. The necessary locking + * is done inside the action class and nothing we need to take care of. + * rgerhards, 2008-10-22 + */ +DEFFUNC_llExecFunc(doHUPActions) +{ + BEGINfunc + actionCallHUPHdlr((action_t*) pData); + ENDfunc + return RS_RET_OK; /* we ignore errors, we can not do anything either way */ +} + + +/* This function processes a HUP after one has been detected. Note that this + * is *NOT* the sighup handler. The signal is recorded by the handler, that record + * detected inside the mainloop and then this function is called to do the + * real work. -- rgerhards, 2008-10-22 + */ +static inline void +doHUP(void) +{ + selector_t *f; + char buf[512]; + + snprintf(buf, sizeof(buf) / sizeof(char), + " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION + "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed, type '%s'.", + (int) myPid, glbl.GetHUPisRestart() ? "restart" : "lightweight"); + errno = 0; + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)buf, 0); + + if(glbl.GetHUPisRestart()) { + DBGPRINTF("Received SIGHUP, configured to be restart, reloading rsyslogd.\n"); + init(); /* main queue is stopped as part of init() */ + } else { + DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); + for(f = Files; f != NULL ; f = f->f_next) { + llExecFunc(&f->llActList, doHUPActions, NULL); + } + } +} + + /* This is the main processing loop. It is called after successful initialization. * When it returns, the syslogd terminates. * Its sole function is to provide some housekeeping things. The real work is done @@ -2634,11 +2675,9 @@ mainloop(void) if(bReduceRepeatMsgs == 1) doFlushRptdMsgs(); - if(restart) { - dbgprintf("\nReceived SIGHUP, reloading rsyslogd.\n"); - /* main queue is stopped as part of init() */ - init(); - restart = 0; + if(bHadHUP) { + doHUP(); + bHadHUP = 0; continue; } } -- cgit From 4f5ab3342c262309ffaed769a5aa9584b7c0e6a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Oct 2008 14:42:06 +0100 Subject: updated support options --- doc/professional_support.html | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/professional_support.html b/doc/professional_support.html index 2cb6c1e1..7724ede8 100644 --- a/doc/professional_support.html +++ b/doc/professional_support.html @@ -49,6 +49,30 @@ configuration file, but will place easy to read comments in the places where you need to put them in. The agreement is governed under German law. You may also purchase this service if you would like to have your own configuration file reviewed, e.g. for auditing purposes.

    +

    Local Installation Support

    +

    If you intend to install rsyslog on your system but would like +to do so with minimal effort and according to your specification, you +can ask us to perform the installation for you. You get a perfect +installation, exactly like you needed, but without a need to +touch the system. This is a perfect choice for the busy administrator! +

    In order to perform this work, we just need ssh access to your +system and the proper permissions. +

    We charge a low one-time fee for this service. For details, please +contact info@adiscon.com. +

    Local Installation Maintenance

    +

    If you used our services to set up the system, why not keep it +running perfectly with maintenance support? Under this contract, we +assure you run a recent build that does not interfere with your +environment and we even carry out change requests you may have. So this +is a hassle-free, everything cared about solution. +

    Again, all we need to have is ssh access and the proper permissions +to your machine. Of course, work will only be carried out when you +expect us to do so. You are always in control of what happens. This +is a perfect outsourcing solution for those who would like to run +a great logging system but can not afford the time to keep it +in perfect shape! +

    We charge a low monthly fee for this service. For details, please +contact info@adiscon.com.

    Custom Development

    Do you need an exotic feature that otherwise would not be implemented? Do you need something really quick, quicker than it is available via @@ -85,4 +109,4 @@ Copyright  Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + -- cgit From 47bbd838bdaa200d89d8210a6a0ba9c322129bca Mon Sep 17 00:00:00 2001 From: Tom Bergfeld Date: Fri, 31 Oct 2008 14:47:30 +0100 Subject: restructured rsyslog.conf documentation Signed-off-by: Rainer Gerhards --- ChangeLog | 1 + doc/expression.html | 3 +- doc/features.html | 13 +- doc/im3195.html | 2 + doc/imfile.html | 2 + doc/imgssapi.html | 2 + doc/imklog.html | 2 + doc/imrelp.html | 2 + doc/imtcp.html | 2 + doc/imuxsock.html | 2 + doc/log_rotation_fix_size.html | 10 + doc/netstream.html | 4 +- doc/omlibdbi.html | 2 + doc/ommail.html | 3 +- doc/ommysql.html | 2 + doc/omrelp.html | 4 +- doc/omsnmp.html | 1 + doc/property_replacer.html | 10 + doc/queues.html | 9 + ...onf1_actionexeconlywhenpreviousissuspended.html | 2 + doc/rsconf1_actionresumeinterval.html | 4 +- doc/rsconf1_allowedsender.html | 2 + doc/rsconf1_controlcharacterescapeprefix.html | 2 + doc/rsconf1_debugprintcfsyslinehandlerlist.html | 4 +- doc/rsconf1_debugprintmodulelist.html | 3 +- doc/rsconf1_debugprinttemplatelist.html | 4 +- doc/rsconf1_dircreatemode.html | 4 +- doc/rsconf1_dirgroup.html | 4 +- doc/rsconf1_dirowner.html | 4 +- ...rsconf1_dropmsgswithmaliciousdnsptrrecords.html | 4 +- doc/rsconf1_droptrailinglfonreception.html | 2 + doc/rsconf1_dynafilecachesize.html | 4 +- doc/rsconf1_escapecontrolcharactersonreceive.html | 2 + doc/rsconf1_failonchownfailure.html | 4 +- doc/rsconf1_filecreatemode.html | 2 + doc/rsconf1_filegroup.html | 4 +- doc/rsconf1_fileowner.html | 4 +- doc/rsconf1_gssforwardservicename.html | 2 + doc/rsconf1_gsslistenservicename.html | 2 + doc/rsconf1_gssmode.html | 2 + doc/rsconf1_includeconfig.html | 4 +- doc/rsconf1_mainmsgqueuesize.html | 2 + doc/rsconf1_markmessageperiod.html | 4 +- doc/rsconf1_moddir.html | 4 +- doc/rsconf1_modload.html | 2 + doc/rsconf1_repeatedmsgreduction.html | 4 +- doc/rsconf1_resetconfigvariables.html | 4 +- doc/rsconf1_umask.html | 4 +- doc/rsyslog_conf.html | 1235 +------------------- doc/rsyslog_high_database_rate.html | 9 + doc/rsyslog_mysql.html | 11 +- doc/rsyslog_ng_comparison.html | 10 + doc/rsyslog_stunnel.html | 11 +- doc/rsyslog_tls.html | 10 + doc/syslog_protocol.html | 9 + 55 files changed, 225 insertions(+), 1239 deletions(-) diff --git a/ChangeLog b/ChangeLog index a11ce27d..0bf28fc4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,7 @@ version before switching to this one. receive loop (aka receiving messsages at a high rate) - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period +- restructured rsyslog.conf documentation --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - consolidated time calls during msg object creation, improves performance diff --git a/doc/expression.html b/doc/expression.html index e7eb7842..9e37cb7a 100644 --- a/doc/expression.html +++ b/doc/expression.html @@ -2,6 +2,7 @@ Expressions +back

    Expressions

    Rsyslog supports expressions at a growing number of places. So far, they are supported for filtering messages.

    Expression support is provided by RainerScript. For now, please see the formal expression definition in RainerScript ABNF. It is the "expr" node.

    C-like comments (/* some comment */) are supported inside the expression, but not yet in the rest of the configuration file.

    [rsyslog.conf overview] @@ -13,4 +14,4 @@ Copyright Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + diff --git a/doc/features.html b/doc/features.html index d221eb77..89796a41 100644 --- a/doc/features.html +++ b/doc/features.html @@ -1,8 +1,8 @@ rsyslog features - +back

    RSyslog - Features

    This page lists both current features as well as those being considered for future versions of rsyslog. If you @@ -134,4 +134,15 @@ future of RFC 3195 in rsyslog.

    To see when each feature was added, see the rsyslog change log (online only).

    + +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + + diff --git a/doc/im3195.html b/doc/im3195.html index d6f2f2ed..aad9f3d1 100644 --- a/doc/im3195.html +++ b/doc/im3195.html @@ -4,6 +4,8 @@ +back +

    RFC3195 Input Module

    Module Name:    im3195

    Author: Rainer Gerhards diff --git a/doc/imfile.html b/doc/imfile.html index 5bdbce5c..af0413dd 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -2,6 +2,8 @@ Text File Input Monitor +back +

    Text File Input Module

    Module Name:    imfile

    Author: Rainer Gerhards diff --git a/doc/imgssapi.html b/doc/imgssapi.html index d644303e..ec183fe7 100644 --- a/doc/imgssapi.html +++ b/doc/imgssapi.html @@ -4,6 +4,8 @@ +back +

    GSSAPI Syslog Input Module

    Module Name:    imgssapi

    Author: varmojfekoj

    diff --git a/doc/imklog.html b/doc/imklog.html index b5b21e84..9166bae6 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -4,6 +4,8 @@ +back +

    Kernel Log Input Module

    Module Name:    imklog

    Author: Rainer Gerhards diff --git a/doc/imrelp.html b/doc/imrelp.html index bfdaad84..53826ac2 100644 --- a/doc/imrelp.html +++ b/doc/imrelp.html @@ -4,6 +4,8 @@ +back +

    RELP Input Module

    Module Name:    imrelp

    Author: Rainer Gerhards

    diff --git a/doc/imtcp.html b/doc/imtcp.html index ecc72748..583cd531 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -2,6 +2,8 @@ TCP Syslog Input Module +back +

    TCP Syslog Input Module

    Module Name:    imtcp

    Author: Rainer Gerhards diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 77491992..472470a0 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -4,6 +4,8 @@ Unix Socket Input +back +

    Unix Socket Input

    Module Name:    imuxsock

    Author: Rainer Gerhards diff --git a/doc/log_rotation_fix_size.html b/doc/log_rotation_fix_size.html index 0b9a3b2e..190b24cb 100644 --- a/doc/log_rotation_fix_size.html +++ b/doc/log_rotation_fix_size.html @@ -3,6 +3,8 @@ +back +

    Log rotation with rsyslog

    Written by Michael Meckelein

    @@ -54,6 +56,14 @@ file and fill it up with new logs. So the latest logs are always in log_roatatio

    With this approach two files for logging are used, each with a maximum size of 50 MB. So we can say we have successfully configured a log rotation which satisfies our requirement. We keep the logs at a fixed-size level of100 MB.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    diff --git a/doc/netstream.html b/doc/netstream.html index e7d54c12..cbfa12ae 100644 --- a/doc/netstream.html +++ b/doc/netstream.html @@ -3,6 +3,8 @@ +back +

    Network Stream Drivers

    Network stream drivers are a layer between various parts of rsyslogd (e.g. the imtcp module) and the transport layer. They provide sequenced delivery, authentication and @@ -18,4 +20,4 @@ Copyright Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + diff --git a/doc/omlibdbi.html b/doc/omlibdbi.html index 8ff74371..ec1d01b6 100644 --- a/doc/omlibdbi.html +++ b/doc/omlibdbi.html @@ -4,6 +4,8 @@ +back +

    Generic Database Output Module (omlibdbi)

    Module Name:    omlibdbi

    Author: Rainer Gerhards diff --git a/doc/ommail.html b/doc/ommail.html index c18cf3f8..0841dc9f 100644 --- a/doc/ommail.html +++ b/doc/ommail.html @@ -1,6 +1,6 @@ mail output module - sending syslog messages via mail - +back

    Mail Output Module (ommail)

    @@ -142,4 +142,5 @@ Copyright © 2008 by Rainer Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    + diff --git a/doc/ommysql.html b/doc/ommysql.html index 79d913eb..7a3f5930 100644 --- a/doc/ommysql.html +++ b/doc/ommysql.html @@ -5,6 +5,8 @@ +back +

    MySQL Database Output Module

    Module Name:    ommysql

    Author: Michael Meckelein (Initial Author) / Rainer Gerhards diff --git a/doc/omrelp.html b/doc/omrelp.html index 0952cc71..82a62afc 100644 --- a/doc/omrelp.html +++ b/doc/omrelp.html @@ -4,6 +4,8 @@ +back +

    RELP Output Module (omlibdbi)

    Module Name:    omrelp

    Author: Rainer Gerhards @@ -51,4 +53,4 @@ Copyright Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + diff --git a/doc/omsnmp.html b/doc/omsnmp.html index 31aaef24..b38a594f 100644 --- a/doc/omsnmp.html +++ b/doc/omsnmp.html @@ -4,6 +4,7 @@ SNMP Output Module +back

    SNMP Output Module

    Module Name:    omsnmp

    diff --git a/doc/property_replacer.html b/doc/property_replacer.html index f666fb76..34e2116c 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -1,6 +1,7 @@ The Rsyslogd Property Replacer +back

    The Property Replacer

    The property replacer is a core component in rsyslogd's output system. A syslog message has a number of @@ -398,4 +399,13 @@ to record severity and facility of a message)

  • Configuration file syntax, this is where you actually use the property replacer.
    • +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/queues.html b/doc/queues.html index 7461121b..727bc26a 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -3,6 +3,7 @@ Understanding rsyslog queues +back

    Understanding rsyslog Queues

    Rsyslog uses queues whenever two activities need to be loosely coupled. With a @@ -357,5 +358,13 @@ environment for the next action).

    parameters, because not all are applicable. For example, in current output module design, actions do not support multi-threading. Consequently, the number of worker threads is fixed to one for action queues and can not be changed.

    +[manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    diff --git a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html index d5cf8b14..1626b4ca 100644 --- a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html +++ b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ActionExecOnlyWhenPreviousIsSuspended

    Type: global configuration directive

    Default: off

    diff --git a/doc/rsconf1_actionresumeinterval.html b/doc/rsconf1_actionresumeinterval.html index a854a212..c0365470 100644 --- a/doc/rsconf1_actionresumeinterval.html +++ b/doc/rsconf1_actionresumeinterval.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ActionResumeInterval

    Type: global configuration directive

    Default: 30

    @@ -27,4 +29,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_allowedsender.html b/doc/rsconf1_allowedsender.html index 4a980b89..ac39e268 100644 --- a/doc/rsconf1_allowedsender.html +++ b/doc/rsconf1_allowedsender.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $AllowedSender

    Type: global configuration directive

    Default: all allowed

    diff --git a/doc/rsconf1_controlcharacterescapeprefix.html b/doc/rsconf1_controlcharacterescapeprefix.html index 6dab1e2e..45cd9230 100644 --- a/doc/rsconf1_controlcharacterescapeprefix.html +++ b/doc/rsconf1_controlcharacterescapeprefix.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ControlCharacterEscapePrefix

    Type: global configuration directive

    Default: \

    diff --git a/doc/rsconf1_debugprintcfsyslinehandlerlist.html b/doc/rsconf1_debugprintcfsyslinehandlerlist.html index 1aad7552..e158de43 100644 --- a/doc/rsconf1_debugprintcfsyslinehandlerlist.html +++ b/doc/rsconf1_debugprintcfsyslinehandlerlist.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DebugPrintCFSyslineHandlerList

    Type: global configuration directive

    Default: on

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_debugprintmodulelist.html b/doc/rsconf1_debugprintmodulelist.html index 4d8e9bff..f25663fb 100644 --- a/doc/rsconf1_debugprintmodulelist.html +++ b/doc/rsconf1_debugprintmodulelist.html @@ -3,6 +3,7 @@ rsyslog.conf file +back

    $DebugPrintModuleList

    Type: global configuration directive

    Default: on

    @@ -19,4 +20,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_debugprinttemplatelist.html b/doc/rsconf1_debugprinttemplatelist.html index 243530e1..b5f1f28f 100644 --- a/doc/rsconf1_debugprinttemplatelist.html +++ b/doc/rsconf1_debugprinttemplatelist.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DebugPrintTemplateList

    Type: global configuration directive

    Default: on

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dircreatemode.html b/doc/rsconf1_dircreatemode.html index 66a35e18..9a9c61eb 100644 --- a/doc/rsconf1_dircreatemode.html +++ b/doc/rsconf1_dircreatemode.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DirCreateMode

    Type: global configuration directive

    Default: 0644

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dirgroup.html b/doc/rsconf1_dirgroup.html index 868e5ecd..de070126 100644 --- a/doc/rsconf1_dirgroup.html +++ b/doc/rsconf1_dirgroup.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DirGroup

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dirowner.html b/doc/rsconf1_dirowner.html index e85a5122..da8e252d 100644 --- a/doc/rsconf1_dirowner.html +++ b/doc/rsconf1_dirowner.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DirOwner

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html index e0a53ae6..95027a70 100644 --- a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html +++ b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DropMsgsWithMaliciousDnsPTRRecords

    Type: global configuration directive

    Default: off

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_droptrailinglfonreception.html b/doc/rsconf1_droptrailinglfonreception.html index 1e3aa8af..fb59b871 100644 --- a/doc/rsconf1_droptrailinglfonreception.html +++ b/doc/rsconf1_droptrailinglfonreception.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DropTrailingLFOnReception

    Type: global configuration directive

    Default: on

    diff --git a/doc/rsconf1_dynafilecachesize.html b/doc/rsconf1_dynafilecachesize.html index 3813f981..cacbf6e5 100644 --- a/doc/rsconf1_dynafilecachesize.html +++ b/doc/rsconf1_dynafilecachesize.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DynaFileCacheSize

    Type: global configuration directive

    Default: 10

    @@ -20,4 +22,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_escapecontrolcharactersonreceive.html b/doc/rsconf1_escapecontrolcharactersonreceive.html index 26917736..178f9a6f 100644 --- a/doc/rsconf1_escapecontrolcharactersonreceive.html +++ b/doc/rsconf1_escapecontrolcharactersonreceive.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $EscapeControlCharactersOnReceive

    Type: global configuration directive

    Default: on

    diff --git a/doc/rsconf1_failonchownfailure.html b/doc/rsconf1_failonchownfailure.html index 0e646e36..d8bbab82 100644 --- a/doc/rsconf1_failonchownfailure.html +++ b/doc/rsconf1_failonchownfailure.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FailOnChownFailure

    Type: global configuration directive

    Default: on

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_filecreatemode.html b/doc/rsconf1_filecreatemode.html index c8440864..10b0317b 100644 --- a/doc/rsconf1_filecreatemode.html +++ b/doc/rsconf1_filecreatemode.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FileCreateMode

    Type: global configuration directive

    Default: 0644

    diff --git a/doc/rsconf1_filegroup.html b/doc/rsconf1_filegroup.html index b9acaab7..dd5b8ad5 100644 --- a/doc/rsconf1_filegroup.html +++ b/doc/rsconf1_filegroup.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FileGroup

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_fileowner.html b/doc/rsconf1_fileowner.html index 7a9cbbc7..935cfffd 100644 --- a/doc/rsconf1_fileowner.html +++ b/doc/rsconf1_fileowner.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FileOwner

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_gssforwardservicename.html b/doc/rsconf1_gssforwardservicename.html index 9d39dc2a..45d9ba98 100644 --- a/doc/rsconf1_gssforwardservicename.html +++ b/doc/rsconf1_gssforwardservicename.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $GssForwardServiceName

    Type: global configuration directive

    Default: host

    diff --git a/doc/rsconf1_gsslistenservicename.html b/doc/rsconf1_gsslistenservicename.html index cd03dc58..5fdf3edc 100644 --- a/doc/rsconf1_gsslistenservicename.html +++ b/doc/rsconf1_gsslistenservicename.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $GssListenServiceName

    Type: global configuration directive

    Default: host

    diff --git a/doc/rsconf1_gssmode.html b/doc/rsconf1_gssmode.html index 71c50696..2b1d5656 100644 --- a/doc/rsconf1_gssmode.html +++ b/doc/rsconf1_gssmode.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $GssMode

    Type: global configuration directive

    Default: encryption

    diff --git a/doc/rsconf1_includeconfig.html b/doc/rsconf1_includeconfig.html index 24462f77..132cee6f 100644 --- a/doc/rsconf1_includeconfig.html +++ b/doc/rsconf1_includeconfig.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $IncludeConfig

    Type: global configuration directive

    Default:

    @@ -43,4 +45,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_mainmsgqueuesize.html b/doc/rsconf1_mainmsgqueuesize.html index acf88e94..ffed1c09 100644 --- a/doc/rsconf1_mainmsgqueuesize.html +++ b/doc/rsconf1_mainmsgqueuesize.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $MainMsgQueueSize

    Type: global configuration directive

    Default: 10000

    diff --git a/doc/rsconf1_markmessageperiod.html b/doc/rsconf1_markmessageperiod.html index 9b6590cd..2c833339 100644 --- a/doc/rsconf1_markmessageperiod.html +++ b/doc/rsconf1_markmessageperiod.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $MarkMessagePeriod

    Type: specific to immark input module

    Default: 1800 (20 minutes)

    @@ -27,4 +29,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_moddir.html b/doc/rsconf1_moddir.html index ced07dc9..889de05d 100644 --- a/doc/rsconf1_moddir.html +++ b/doc/rsconf1_moddir.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ModDir

    Type: global configuration directive

    Default: system default for user libraries, e.g. @@ -24,4 +26,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_modload.html b/doc/rsconf1_modload.html index a2b8087a..ce457ea5 100644 --- a/doc/rsconf1_modload.html +++ b/doc/rsconf1_modload.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ModLoad

    Type: global configuration directive

    Default:

    diff --git a/doc/rsconf1_repeatedmsgreduction.html b/doc/rsconf1_repeatedmsgreduction.html index 20e56f89..248e8343 100644 --- a/doc/rsconf1_repeatedmsgreduction.html +++ b/doc/rsconf1_repeatedmsgreduction.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $RepeatedMsgReduction

    Type: global configuration directive

    Default: depending on -e

    @@ -20,4 +22,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_resetconfigvariables.html b/doc/rsconf1_resetconfigvariables.html index 9794d158..46cf0bdf 100644 --- a/doc/rsconf1_resetconfigvariables.html +++ b/doc/rsconf1_resetconfigvariables.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ResetConfigVariables

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_umask.html b/doc/rsconf1_umask.html index ee47dbad..8e41e672 100644 --- a/doc/rsconf1_umask.html +++ b/doc/rsconf1_umask.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $UMASK

    Type: global configuration directive

    Default:

    @@ -21,4 +23,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index cd3db405..852d95b5 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -20,257 +20,13 @@ possible. While, for obvious reasons, enhanced features require a different config file syntax, rsyslogd should be able to work with a standard syslog.conf file. This is especially useful while you are migrating from syslogd to rsyslogd.

    -

    Modules

    -

    Rsyslog has a modular design. Consequently, there is a growing -number of modules. Here is the entry point to their documentation and -what they do (list is currently not complete)

    -
      -
    • omsnmp - SNMP -trap output module
      • -
    • omrelp - RELP -output module
      • -
    • omgssapi - output module for GSS-enabled syslog
      • -
    • ommysql - output module for MySQL
      • -
    • ompgsql - output module for PostgreSQL
      • -
    • omlibdbi - -generic database output module (Firebird/Interbase, MS SQL, Sybase, -SQLLite, Ingres, Oracle, mSQL)
      • -
    • ommail - -permits rsyslog to alert folks by mail if something important happens
      • -
    • imfile --  input module for text files
      • -
    • imrelp - RELP -input module
      • -
    • imudp - udp syslog message input
      • -
    • imtcp - input -plugin for plain tcp syslog
      • -
    • imgssapi - -input plugin for plain tcp and GSS-enabled syslog
      • -
    • immark - support for mark messages
      • -
    • imklog - kernel logging
      • -
    • imuxsock - -unix sockets, including the system log socket
      • -
    • im3195 - -accepts syslog messages via RFC 3195
      • -
    -

    Please note that each module provides configuration -directives, which are NOT necessarily being listed below. Also -remember, that a modules configuration directive (and functionality) is -only available if it has been loaded (using $ModLoad).

    +

    Modules

    Lines

    Lines can be continued by specifying a backslash ("\") as the last character of the line. There is a hard-coded maximum line length of 4K. If you need lines larger than that, you need to change compile-time settings inside rsyslog and recompile. -

    Global Directives

    -

    All global directives need to be specified on a line by their -own and must start with a dollar-sign. Here is a list in alphabetical -order. Follow links for a description.

    -

    Please note that not all directives here are actually global. Some affect -only the next action. This documentation will be changed soon. -

    Not all directives have an in-depth description right now. -Default values for them are in bold. A more in-depth description will -appear as implementation progresses. -

    -

    Be sure to read information about queues in rsyslog - -many parameter settings modify queue parameters. If in doubt, use the -default, it is usually well-chosen and applicable in most cases.

    -
      -
    • $ActionExecOnlyWhenPreviousIsSuspended
      • -
    • $ActionExecOnlyOnceEveryInterval <seconds> - -execute action only if the last execute is at last -<seconds> seconds in the past (more info in ommail, -but may be used with any action)
      • -
    • $ActionExecOnlyEveryNthTime <number> - If configured, the next action will -only be executed every n-th time. For example, if configured to 3, the first two messages -that go into the action will be dropped, the 3rd will actually cause the action to execute, -the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note: -this setting is automatically re-set when the actual action is defined.
      • -
    • $ActionExecOnlyEveryNthTimeTimeout <number-of-seconds> - has a meaning only if -$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout -setting specifies after which period the counting of "previous actions" expires and -a new action count is begun. Specify 0 (the default) to disable timeouts. -
      -Why is this option needed? Consider this case: a message comes in at, eg., 10am. That's -count 1. Then, nothing happens for the next 10 hours. At 8pm, the next -one occurs. That's count 2. Another 5 hours later, the next message -occurs, bringing the total count to 3. Thus, this message now triggers -the rule. -
      -The question is if this is desired behavior? Or should the rule only be -triggered if the messages occur within an e.g. 20 minute window? If the -later is the case, you need a -
      -$ActionExecOnlyEveryNthTimeTimeout 1200 -
      -This directive will timeout previous messages seen if they are older -than 20 minutes. In the example above, the count would now be always 1 -and consequently no rule would ever be triggered. - -
    • $ActionFileDefaultTemplate [templateName] - sets a new default template for file actions
      • -
    • $ActionFileEnableSync [on/off] - enables file -syncing capability of omfile
      • -
    • $ActionForwardDefaultTemplate [templateName] - sets a new -default template for UDP and plain TCP forwarding action
      • -
    • $ActionGSSForwardDefaultTemplate [templateName] - sets a -new default template for GSS-API forwarding action
      • -
    • $ActionQueueCheckpointInterval <number>
      • -
    • $ActionQueueDequeueSlowdown <number> [number -is timeout in microseconds (1000000us is 1sec!), -default 0 (no delay). Simple rate-limiting!]
      • -
    • $ActionQueueDiscardMark <number> [default -9750]
      • -
    • $ActionQueueDiscardSeverity <number> -[*numerical* severity! default 4 (warning)]
      • -
    • $ActionQueueFileName <name>
      • -
    • $ActionQueueHighWaterMark <number> [default -8000]
      • -
    • $ActionQueueImmediateShutdown [on/off]
      • -
    • $ActionQueueSize <number>
      • -
    • $ActionQueueLowWaterMark <number> [default -2000]
      • -
    • $ActionQueueMaxFileSize <size_nbr>, default 1m
      • -
    • $ActionQueueTimeoutActionCompletion <number> -[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means -immediate!]
      • -
    • $ActionQueueTimeoutEnqueue <number> [number -is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
      • -
    • $ActionQueueTimeoutShutdown <number> [number -is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
      • -
    • $ActionQueueWorkerTimeoutThreadShutdown -<number> [number is timeout in ms (1000ms is 1sec!), -default 60000 (1 minute)]
      • -
    • $ActionQueueType [FixedArray/LinkedList/Direct/Disk]
      • -
    • $ActionQueueSaveOnShutdown  [on/off] -
      • -
    • $ActionQueueWorkerThreads <number>, num worker threads, default 1, recommended 1
      • -
    • $ActionQueueWorkerThreadMinumumMessages <number>, default 100
      • -
    • $ActionResumeInterval
      • -
    • $ActionResumeRetryCount <number> [default 0, -1 means eternal]
      • -
    • $ActionSendResendLastMsgOnReconn <[on/off]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. -
    • $ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action -
    • $ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver -(driver-specific)
    • $ActionSendStreamDriverAuthMode <mode>,  authentication mode to use with the stream driver -(driver-specific)
    • $ActionSendStreamDriverPermittedPeer <ID>,  accepted fingerprint (SHA1) or name of remote peer -(driver-specific) - directive may go away!
      • -
    • $AllowedSender
      • -
    • $ControlCharacterEscapePrefix
      • -
    • $DebugPrintCFSyslineHandlerList
      • - -
    • $DebugPrintModuleList
      • -
    • $DebugPrintTemplateList
      • -
    • $DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
      • -
    • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
      • -
    • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
      • -
    • $DirCreateMode
      • -
    • $DirGroup
      • -
    • $DirOwner
      • -
    • $DropMsgsWithMaliciousDnsPTRRecords
      • -
    • $DropTrailingLFOnReception
      • -
    • $DynaFileCacheSize
      • -
    • $EscapeControlCharactersOnReceive
      • -
    • $ErrorMessagesToStderr [on|off] - direct rsyslogd error message to stderr (in addition to other targets)
      • -
    • $FailOnChownFailure
      • -
    • $FileCreateMode
      • -
    • $FileGroup
      • -
    • $FileOwner
      • -
    • $GssForwardServiceName
      • -
    • $GssListenServiceName
      • -
    • $GssMode
      • -
    • $HUPisRestart [on/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending -on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is -not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for -compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually -the only one that is needed e.g. for log rotation. It is recommended to set the setting to "off".
      • -
    • $IncludeConfig
    • MainMsgQueueCheckpointInterval <number>
      • -
    • $MainMsgQueueDequeueSlowdown <number> [number -is timeout in microseconds (1000000us is 1sec!), -default 0 (no delay). Simple rate-limiting!]
      • -
    • $MainMsgQueueDiscardMark <number> [default 9750]
      • -
    • $MainMsgQueueDiscardSeverity <severity> -[either a textual or numerical severity! default 4 (warning)]
      • -
    • $MainMsgQueueFileName <name>
      • -
    • $MainMsgQueueHighWaterMark <number> [default -8000]
      • -
    • $MainMsgQueueImmediateShutdown [on/off]
      • -
    • $MainMsgQueueSize
      • -
    • $MainMsgQueueLowWaterMark <number> [default -2000]
      • -
    • $MainMsgQueueMaxFileSize <size_nbr>, default -1m
      • -
    • $MainMsgQueueTimeoutActionCompletion -<number> [number is timeout in ms (1000ms is 1sec!), -default -1000, 0 means immediate!]
      • -
    • $MainMsgQueueTimeoutEnqueue <number> [number -is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
      • -
    • $MainMsgQueueTimeoutShutdown <number> [number -is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
      • -
    • $MainMsgQueueWorkerTimeoutThreadShutdown -<number> [number is timeout in ms (1000ms is 1sec!), -default 60000 (1 minute)]
      • -
    • $MainMsgQueueType [FixedArray/LinkedList/Direct/Disk]
      • -
    • $MainMsgQueueSaveOnShutdown  [on/off] -
      • -
    • $MainMsgQueueWorkerThreads <number>, num -worker threads, default 1, recommended 1
      • -
    • $MainMsgQueueWorkerThreadMinumumMessages <number>, default 100
      • -
    • $MarkMessagePeriod (immark)
      • -
    • $MaxMessageSize <size_nbr>, default 2k - allows to specify maximum supported message size -(both for sending and receiving). The default -should be sufficient for almost all cases. Do not set this below 1k, as it would cause -interoperability problems with other syslog implementations.
      -Change the setting to e.g. 32768 if you would like to -support large message sizes for IHE (32k is the current maximum -needed for IHE). I was initially tempted to set the default to 32k, -but there is a some memory footprint with the current -implementation in rsyslog. -
      If you intend to receive Windows Event Log data (e.g. via -EventReporter), you might want to -increase this number to an even higher value, as event -log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea). -Note: testing showed that 4k seems to be -the typical maximum for UDP based syslog. This is an IP stack -restriction. Not always ... but very often. If you go beyond -that value, be sure to test that rsyslogd actually does what -you think it should do ;) It is highly suggested to use a TCP based transport -instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions. -
      Note that 2k, the current default, is the smallest size that must be -supported in order to be compliant to the upcoming new syslog RFC series. -
      • -
    • $ModDir
      • -
    • $ModLoad
      • -
    • $RepeatedMsgReduction
      • -
    • $ResetConfigVariables
      • -
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better -performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The -default may change as uniprocessor systems become less common.
      • -
    • $WorkDirectory <name> (directory for spool and other work files)
      • -
    • $UDPServerAddress <IP> (imudp) -- local IP -address (or name) the UDP listens should bind to
      • -
    • $UDPServerRun <port> (imudp) -- former --r<port> option, default 514, start UDP server on this -port, "*" means all addresses
      • -
    • $UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance -optimization. Getting the system time is very costly. With this setting, imudp can -be instructed to obtain the precise time only once every n-times. This logic is -only activated if messages come in at a very fast rate, so doing less frequent -time calls should usually be acceptable. The default value is two, because we have -seen that even without optimization the kernel often returns twice the identical time. -You can set this value as high as you like, but do so at your own risk. The higher -the value, the less precise the timestamp. -
    • $UMASK
      • -
    -

    Where <size_nbr> is specified above, -modifiers can be used after the number part. For example, 1k means -1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa). -Lower case letters refer to the traditional binary defintion (e.g. 1m -equals 1,048,576) whereas upper case letters refer to their new -1000-based definition (e.g 1M equals 1,000,000).

    -

    Numbers may include '.' and ',' for readability. So you can -for example specify either "1000" or "1,000" with the same result. -Please note that rsyslogd simply ignores the punctuation. Form it's -point of view, "1,,0.0.,.,0" also has the value 1000.

    +

    Global Directives

    Basic Structure

    Rsyslog supports standard sysklogd's configuration file format and extends it. So in general, you can take a "normal" syslog.conf and @@ -289,975 +45,15 @@ priorities belonging to the specified action.

    Lines starting with a hash mark ("#'') and empty lines are ignored.

    -

    Templates

    -

    Templates are a key feature of rsyslog. They allow to specify -any -format a user might want. They are also used for dynamic file name -generation. Every output in rsyslog uses templates - this holds true -for files, user messages and so on. The database writer expects its -template to be a proper SQL statement - so this is highly customizable -too. You might ask how does all of this work when no templates at all -are specified. Good question ;) The answer is simple, though. Templates -compatible with the stock syslogd formats are hardcoded into rsyslogd. -So if no template is specified, we use one of these hardcoded -templates. Search for "template_" in syslogd.c and you will find the -hardcoded ones.

    -

    A template consists of a template directive, a name, the -actual template text and optional options. A sample is:

    -
    $template MyTemplateName,"\7Text -%property% some more text\n",<options>
    -

    The "$template" is the template directive. It tells rsyslog -that this line contains a template. "MyTemplateName" is the template -name. All -other config lines refer to this name. The text within quotes is the -actual template text. The backslash is an escape character, much as it -is in C. It does all these "cool" things. For example, \7 rings the -bell (this is an ASCII value), \n is a new line. C programmers and perl -coders have the advantage of knowing this, but the set in rsyslog is a -bit restricted currently. -

    -

    All text in the template is used literally, except for things -within percent signs. These are properties and allow you access to the -contents of the syslog message. Properties are accessed via the -property replacer (nice name, huh) and it can do cool things, too. For -example, it can pick a substring or do date-specific formatting. More -on this is below, on some lines of the property replacer.
    -
    -The <options> part is optional. It carries options -influencing the template as whole. See details below. Be sure NOT to -mistake template options with property options - the later ones are -processed by the property replacer and apply to a SINGLE property, only -(and not the whole template).
    -
    -Template options are case-insensitive. Currently defined are:

    -

    sql - format the string suitable for a SQL -statement in MySQL format. This will replace single quotes ("'") and -the backslash character by their backslash-escaped counterpart ("\'" -and "\\") inside each field. Please note that in MySQL configuration, -the NO_BACKSLASH_ESCAPES -mode must be turned off for this format to work (this is the default).

    -

    stdsql - format the string suitable for a -SQL statement that is to be sent to a standards-compliant sql server. -This will replace single quotes ("'") by two single quotes ("''") -inside each field. You must use stdsql together with MySQL if in MySQL -configuration the -NO_BACKSLASH_ESCAPES is -turned on.

    -

    Either the sql or stdsql  -option must be specified when a template is used -for writing to a database, otherwise injection might occur. Please note -that due to the unfortunate fact that several vendors have violated the -sql standard and introduced their own escape methods, it is impossible -to have a single option doing all the work.  So you yourself -must make sure you are using the right format. If you choose -the wrong one, you are still vulnerable to sql injection.
    -
    -Please note that the database writer *checks* that the sql option is -present in the template. If it is not present, the write database -action is disabled. This is to guard you against accidental forgetting -it and then becoming vulnerable to SQL injection. The sql option can -also be useful with files - especially if you want to import them into -a database on another machine for performance reasons. However, do NOT -use it if you do not have a real need for it - among others, it takes -some toll on the processing time. Not much, but on a really busy system -you might notice it ;)

    -

    The default template for the write to database action has the -sql option set. As we currently support only MySQL and the sql option -matches the default MySQL configuration, this is a good choice. -However, if you have turned on -NO_BACKSLASH_ESCAPES in -your MySQL config, you need to supply a template with the stdsql -option. Otherwise you will become vulnerable to SQL injection.
    -
    -To escape:
    -% = \%
    -\ = \\ --> '\' is used to escape (as in C)
    -$template TraditionalFormat,%timegenerated% %HOSTNAME% -%syslogtag%%msg%\n"
    -
    -Properties can be accessed by the property -replacer (see there for details).

    -

    Please note that templates can also by -used to generate selector lines with dynamic file names. For -example, if you would like to split syslog messages from different -hosts to different files (one per host), you can define the following -template:

    -
    $template -DynFile,"/var/log/system-%HOSTNAME%.log"
    -

    This template can then be used when defining an output -selector line. It will result in something like -"/var/log/system-localhost.log"

    -

    Template -names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT -use them if, otherwise you may receive a conflict in the future (and -quite unpredictable behaviour). There is a small set of pre-defined -templates that you can use without the need to define it:

    -
      -
    • RSYSLOG_TraditionalFileFormat -- the "old style" default log file format with low-precision timestamps
      • -
    • RSYSLOG_FileFormat -- a modern-style logfile format similar to TraditionalFileFormat, buth -with high-precision timestamps and timezone information
      • -
    • RSYSLOG_TraditionalForwardFormat -- the traditional forwarding format with low-precision timestamps. Most -useful if you send messages to other syslogd's or rsyslogd -below -version 3.12.5.
      • -
    • RSYSLOG_ForwardFormat -- a new high-precision forwarding format very similar to the -traditional one, but with high-precision timestamps and timezone -information. Recommended to be used when sending messages to rsyslog -3.12.5 or above.
      • -
    • RSYSLOG_SyslogProtocol23Format -- the format specified in IETF's internet-draft -ietf-syslog-protocol-23, which is assumed to be come the new syslog -standard RFC. This format includes several improvements. The rsyslog -message parser understands this format, so you can use it together with -all relatively recent versions of rsyslog. Other syslogd's may get -hopelessly confused if receiving that format, so check before you use -it. Note that the format is unlikely to change when the final RFC comes -out, but this may happen.
      • -
    • RSYSLOG_DebugFormat -- a special format used for troubleshooting property problems. This format -is meant to be written to a log file. Do not use for production or remote -forwarding.
      • -
    -

    Output Channels

    -

    Output Channels are a new concept first introduced in rsyslog -0.9.0. As of this writing, it is most likely that they will -be replaced by something different in the future. So if you -use them, be prepared to change you configuration file syntax when you -upgrade to a later release.
    -
    -The idea behind output channel definitions is that it shall provide an -umbrella for any type of output that the user might want. In essence,
    -this is the "file" part of selector lines (and this is why we are not -sure output channel syntax will stay after the next review). There is a
    -difference, though: selector channels both have filter conditions -(currently facility and severity) as well as the output destination. -Output channels define the output definition, only. As of this build, -they can only be used to write to files - not pipes, ttys or whatever -else. If we stick with output channels, this will change over time.

    -

    In concept, an output channel includes everything needed to -know about an output actions. In practice, the current implementation -only carries
    -a filename, a maximum file size and a command to be issued when this -file size is reached. More things might be present in future version, -which might also change the syntax of the directive.

    -

    Output channels are defined via an $outchannel directive. It's -syntax is as follows:
    -
    -$outchannel name,file-name,max-size,action-on-max-size
    -
    -name is the name of the output channel (not the file), file-name is the -file name to be written to, max-size the maximum allowed size and -action-on-max-size a command to be issued when the max size is reached. -This command always has exactly one parameter. The binary is that part -of action-on-max-size before the first space, its parameter is -everything behind that space.
    -
    -Please note that max-size is queried BEFORE writing the log message to -the file. So be sure to set this limit reasonably low so that any -message might fit. For the current release, setting it 1k lower than -you expected is helpful. The max-size must always be specified in bytes -- there are no special symbols (like 1k, 1m,...) at this point of -development.
    -
    -Keep in mind that $outchannel just defines a channel with "name". It -does not activate it. To do so, you must use a selector line (see -below). That selector line includes the channel name plus an $ sign in -front of it. A sample might be:
    -
    -*.* $mychannel
    -
    -In its current form, output channels primarily provide the ability to -size-limit an output file. To do so, specify a maximum size. When this -size is reached, rsyslogd will execute the action-on-max-size command -and then reopen the file and retry. The command should be something -like a log rotation -script or a similar thing.

    -

    If there is no action-on-max-size command or the command did -not resolve the situation, the file is closed and never reopened by -rsyslogd (except, of course, by huping it). This logic was integrated -when we first experienced severe issues with files larger 2gb, which -could lead to rsyslogd dumping core. In such cases, it is more -appropriate to stop writing to a single file. Meanwhile, rsyslogd has -been fixed to support files larger 2gb, but obviously only on file -systems and operating system versions that do so. So it can still make -sense to enforce a 2gb file size limit.

    -

    Filter Conditions

    -

    Rsyslog offers four different types "filter conditions":

    -
      -
    • BSD-style blocks
      • -
    • "traditional" severity and facility based selectors
      • -
    • property-based filters
      • -
    • expression-based filters
      • -
    -

    Blocks

    -

    Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each -block of lines is separated from the previous block by a program or -hostname specification. A block will only log messages corresponding to -the most recent program and hostname specifications given. Thus, a -block which selects ‘ppp’ as the program, directly followed by a block -that selects messages from the hostname ‘dialhost’, then the second -block will only log messages from the ppp program on dialhost. -

    -

    A program specification is a line beginning with ‘!prog’ and -the following blocks will be associated with calls to syslog from that -specific program. A program specification for ‘foo’ will also match any -message logged by the kernel with the prefix ‘foo: ’. Alternatively, a -program specification ‘-foo’ causes the following blocks to be applied -to messages from any program but the one specified. A hostname -specification of the form ‘+hostname’ and the following blocks will be -applied to messages received from the specified hostname. -Alternatively, a hostname specification ‘-hostname’ causes the -following blocks to be applied to messages from any host but the one -specified. If the hostname is given as ‘@’, the local hostname will be -used. (NOT YET IMPLEMENTED) A program or hostname specification may be -reset by giving the program or hostname as ‘*’.

    -

    Please note that the "#!prog", "#+hostname" and "#-hostname" -syntax available in BSD syslogd is not supported by rsyslogd. By -default, no hostname or program is set.

    -

    Selectors

    -

    Selectors are the traditional way of filtering syslog -messages. They have been kept in rsyslog with their original -syntax, because it is well-known, highly effective and also needed for -compatibility with stock syslogd configuration files. If you just need -to filter based on priority and facility, you should do this with -selector lines. They are not second-class citizens -in rsyslog and offer the best performance for this job.

    -

    The selector field itself again consists of two parts, a -facility and a priority, separated by a period (".''). Both parts are -case insensitive and can also be specified as decimal numbers, but -don't do that, you have been warned. Both facilities and priorities are -described in rsyslog(3). The names mentioned below correspond to the -similar LOG_-values in /usr/include/rsyslog.h.
    -
    -The facility is one of the following keywords: auth, authpriv, cron, -daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, -user, uucp and local0 through local7. The keyword security should not -be used anymore and mark is only for internal use and therefore should -not be used in applications. Anyway, you may want to specify and -redirect these messages here. The facility specifies the subsystem that -produced the message, i.e. all mail programs log with the mail facility -(LOG_MAIL) if they log using syslog.
    -
    -The priority is one of the following keywords, in ascending order: -debug, info, notice, warning, warn (same as warning), err, error (same -as err), crit, alert, emerg, panic (same as emerg). The keywords error, -warn and panic are deprecated and should not be used anymore. The -priority defines the severity of the message.
    -
    -The behavior of the original BSD syslogd is that all messages of the -specified priority and higher are logged according to the given action. -Rsyslogd behaves the same, but has some extensions.
    -
    -In addition to the above mentioned names the rsyslogd(8) understands -the following extensions: An asterisk ("*'') stands for all facilities -or all priorities, depending on where it is used (before or after the -period). The keyword none stands for no priority of the given facility.
    -
    -You can specify multiple facilities with the same priority pattern in -one statement using the comma (",'') operator. You may specify as much -facilities as you want. Remember that only the facility part from such -a statement is taken, a priority part would be skipped.

    -

    Multiple selectors may be specified for a single action using -the semicolon (";'') separator. Remember that each selector in the -selector field is capable to overwrite the preceding ones. Using this -behavior you can exclude some priorities from the pattern.

    -

    Rsyslogd has a syntax extension to the original BSD source, -that makes its use more intuitively. You may precede every priority -with an equation sign ("='') to specify only this single priority and -not any of the above. You may also (both is valid, too) precede the -priority with an exclamation mark ("!'') to ignore all that -priorities, either exact this one or this and any higher priority. If -you use both extensions than the exclamation mark must occur before the -equation sign, just use it intuitively.

    -

    Property-Based Filters

    -

    Property-based filters are unique to rsyslogd. They allow to -filter on any property, like HOSTNAME, syslogtag and msg. A list of all -currently-supported properties can be found in the property replacer documentation -(but keep in mind that only the properties, not the replacer is -supported). With this filter, each properties can be checked against a -specified value, using a specified compare operation. Currently, there -is only a single compare operation (contains) available, but additional -operations will be added in the future.

    -

    A property-based filter must start with a colon in column 0. -This tells rsyslogd that it is the new filter type. The colon must be -followed by the property name, a comma, the name of the compare -operation to carry out, another comma and then the value to compare -against. This value must be quoted. There can be spaces and tabs -between the commas. Property names and compare operations are -case-sensitive, so "msg" works, while "MSG" is an invalid property -name. In brief, the syntax is as follows:

    -

    :property, [!]compare-operation, "value"

    -

    The following compare-operations are -currently supported:

    - - - - - - - - - - - - - - - - - - - -
    containsChecks if the string provided in value is contained in -the property. There must be an exact match, wildcards are not supported.
    isequalCompares the "value" string provided and the property -contents. These two values must be exactly equal to match. The -difference to contains is that contains searches for the value anywhere -inside the property value, whereas all characters must be identical for -isequal. As such, isequal is most useful for fields like syslogtag or -FROMHOST, where you probably know the exact contents.
    startswithChecks if the value is found exactly at the beginning -of the property value. For example, if you search for "val" with -

    :msg, startswith, "val"

    -

    it will be a match if msg contains "values are in this -message" but it won't match if the msg contains "There are values in -this message" (in the later case, contains would match). Please note -that "startswith" is by far faster than regular expressions. So even -once they are implemented, it can make very much sense -(performance-wise) to use "startswith".

    -
    regexCompares the property against the provided POSIX -regular -expression.
    -

    You can use the bang-character (!) immediately in front of a -compare-operation, the outcome of this operation is negated. For -example, if msg contains "This is an informative message", the -following sample would not match:

    -

    :msg, contains, "error"

    -

    but this one matches:

    -

    :msg, !contains, "error"

    -

    Using negation can be useful if you would like to do some -generic processing but exclude some specific events. You can use the -discard action in conjunction with that. A sample would be:

    -

    *.* -/var/log/allmsgs-including-informational.log
    -:msg, contains, "informational"  ~ -
    -*.* /var/log/allmsgs-but-informational.log

    -

    Do not overlook the red tilde in line 2! In this sample, all -messages are written to the file allmsgs-including-informational.log. -Then, all messages containing the string "informational" are discarded. -That means the config file lines below the "discard line" (number 2 in -our sample) will not be applied to this message. Then, all remaining -lines will also be written to the file allmsgs-but-informational.log.

    -

    Value is a quoted string. It supports some -escape sequences:

    -

    \" - the quote character (e.g. "String with \"Quotes\"")
    -\\ - the backslash character (e.g. "C:\\tmp")

    -

    Escape sequences always start with a backslash. Additional -escape sequences might be added in the future. Backslash characters must -be escaped. Any other sequence then those outlined above is invalid and -may lead to unpredictable results.

    -

    Probably, "msg" is the most prominent use case of property -based filters. It is the actual message text. If you would like to -filter based on some message content (e.g. the presence of a specific -code), this can be done easily by:

    -

    :msg, contains, "ID-4711"

    -

    This filter will match when the message contains the string -"ID-4711". Please note that the comparison is case-sensitive, so it -would not match if "id-4711" would be contained in the message.

    -

    :msg, regex, "fatal .* error"

    -

    This filter uses a POSIX regular expression. It matches when -the -string contains the words "fatal" and "error" with anything in between -(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as -two spaces are required by the regular expression!).

    -

    Getting property-based filters right can sometimes be -challenging. In order to help you do it with as minimal effort as -possible, rsyslogd spits out debug information for all property-based -filters during their evaluation. To enable this, run rsyslogd in -foreground and specify the "-d" option.

    -

    Boolean operations inside property based filters (like -'message contains "ID17" or message contains "ID18"') are currently not -supported (except for "not" as outlined above). Please note that while -it is possible to query facility and severity via property-based -filters, it is far more advisable to use classic selectors (see above) -for those cases.

    -

    Expression-Based Filters

    -Expression based filters allow -filtering on arbitrary complex expressions, which can include boolean, -arithmetic and string operations. Expression filters will evolve into a -full configuration scripting language. Unfortunately, their syntax will -slightly change during that process. So if you use them now, you need -to be prepared to change your configuration files some time later. -However, we try to implement the scripting facility as soon as possible -(also in respect to stage work needed). So the window of exposure is -probably not too long.
    -
    -Expression based filters are indicated by the keyword "if" in column 1 -of a new line. They have this format:
    -
    -if expr then action-part-of-selector-line
    -
    -"If" and "then" are fixed keywords that mus be present. "expr" is a -(potentially quite complex) expression. So the expression documentation for -details. "action-part-of-selector-line" is an action, just as you know -it (e.g. "/var/log/logfile" to write to that file).
    -
    -A few quick samples:
    -
    - -*.* /var/log/file1 # the traditional way
    -if $msg contains 'error' /var/log/errlog # the expression-based way
    -     -
    -Right now, you need to specify numerical values if you would like to -check for facilities and severity. These can be found in RFC 3164. -If you don't like that, you can of course also use the textual property -- just be sure to use the right one. As expression support is enhanced, -this will change. For example, if you would like to filter on message -that have facility local0, start with "DEVNAME" and have either -"error1" or "error0" in their message content, you could use the -following filter:
    -
    - -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog
    -     -
    -Please note that the above must -all be on one line! And if you would like to store all -messages except those that contain "error1" or "error0", you just need -to add a "not":
    -
    - -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and not -($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog
    -     -
    -If you would like to do case-insensitive comparisons, use -"contains_i" instead of "contains" and "startswith_i" instead of -"startswith".
    -
    -Note that regular expressions are currently NOT -supported in expression-based filters. These will be added later when -function support is added to the expression engine (the reason is that -regular expressions will be a separate loadable module, which requires -some more prequisites before it can be implemented).
    -

    ACTIONS

    -

    The action field of a rule describes what to do with the -message. In general, message content is written to a kind of "logfile". -But also other actions might be done, like writing to a database table -or forwarding to another host.
    -
    -Templates can be used with all actions. If used, the specified template -is used to generate the message content (instead of the default -template). To specify a template, write a semicolon after the action -value immediately followed by the template name.
    -
    -Beware: templates MUST be defined BEFORE they are used. It is OK to -define some templates, then use them in selector lines, define more -templates and use use them in the following selector lines. But it is -NOT permitted to use a template in a selector line that is above its -definition. If you do this, the action will be ignored.

    -

    You can have multiple actions for a single selector  (or -more precisely a single filter of such a selector line). Each action -must be on its own line and the line must start with an ampersand -('&') character and have no filters. An example would be

    -

    *.=crit rger
    -& root
    -& /var/log/critmsgs

    -

    These three lines send critical messages to the user rger and -root and also store them in /var/log/critmsgs. Using multiple -actions per selector is convenient and also offers -a performance benefit. As the filter needs to be evaluated -only once, there is less computation required to process the directive -compared to the otherwise-equal config directives below:

    -

    *.=crit rger
    -*.=crit root
    -*.=crit /var/log/critmsgs

    -

     

    -

    Regular File

    -

    Typically messages are logged to real files. The file has to -be specified with full pathname, beginning with a slash "/''.
    -
    -You may prefix each entry with the minus "-'' sign to omit syncing the -file after every logging. Note that you might lose information if the -system crashes right behind a write attempt. Nevertheless this might -give you back some performance, especially if you run programs that use -logging in a very verbose manner.

    -

    If your system is connected to a reliable UPS and you receive -lots of log data (e.g. firewall logs), it might be a very good idea to -turn of -syncing by specifying the "-" in front of the file name.

    -

    The filename can be either static (always -the same) or dynamic (different based on message -received). The later is useful if you would automatically split -messages into different files based on some message criteria. For -example, dynamic file name selectors allow you to split messages into -different files based on the host that sent them. With dynamic file -names, everything is automatic and you do not need any filters.

    -

    It works via the template system. First, you define a template -for the file name. An example can be seen above in the description of -template. We will use the "DynFile" template defined there. Dynamic -filenames are indicated by specifying a questions mark "?" instead of a -slash, followed by the template name. Thus, the selector line for our -dynamic file name would look as follows:

    -
    -*.* ?DynFile -
    -

    That's all you need to do. Rsyslog will now automatically -generate file names for you and store the right messages into the right -files. Please note that the minus sign also works with dynamic file -name selectors. Thus, to avoid syncing, you may use

    -
    -*.* -?DynFile
    -

    And of course you can use templates to specify the output -format:

    -
    -*.* ?DynFile;MyTemplate
    -

    A word of caution: rsyslog creates files as -needed. So if a new host is using your syslog server, rsyslog will -automatically create a new file for it.

    -

    Creating directories is also supported. For -example you can use the hostname as directory and the program name as -file name:

    -
    -$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"
    -

    Named Pipes

    -

    This version of rsyslogd(8) has support for logging output to -named pipes (fifos). A fifo or named pipe can be used as a destination -for log messages by prepending a pipe symbol ("|'') to the name of the -file. This is handy for debugging. Note that the fifo must be created -with the mkfifo(1) command before rsyslogd(8) is started.

    -

    Terminal and Console

    -

    If the file you specified is a tty, special tty-handling is -done, same with /dev/console.

    -

    Remote Machine

    -

    Rsyslogd provides full remote logging, i.e. is able to send -messages to a remote host running rsyslogd(8) and to receive messages -from remote hosts. Using this feature you're able to control all syslog -messages on one host, if all other machines will log remotely to that. -This tears down
    -administration needs.
    -
    -Please note that this version of rsyslogd by default does NOT -forward messages it has received from the network to another host. -Specify the "-h" option to enable this.

    -

    To forward messages to another host, prepend the hostname with -the at sign ("@"). A single at sign means that messages will -be forwarded via UDP protocol (the standard for syslog). If you prepend -two at signs ("@@"), the messages will be transmitted via TCP. Please -note that plain TCP based syslog is not officially standardized, but -most major syslogds support it (e.g. syslog-ng or WinSyslog). The -forwarding action indicator (at-sign) can be followed by one or more -options. If they are given, they must be immediately (without a space) -following the final at sign and be enclosed in parenthesis. The -individual options must be separated by commas. The following options -are right now defined:

    - - - - - - - - - - - -
    -

    z<number>

    -    		Enable zlib-compression for the message. The -<number> is the compression level. It can be 1 (lowest -gain, lowest CPU overhead) to 9 (maximum compression, highest CPU -overhead). The level can also be 0, which means "no compression". If -given, the "z" option is ignored. So this does not make an awful lot of -sense. There is hardly a difference between level 1 and 9 for typical -syslog messages. You can expect a compression gain between 0% and 30% -for typical messages. Very chatty messages may compress up to 50%, but -this is seldom seen with typically traffic. Please note that rsyslogd -checks the compression gain. Messages with 60 bytes or less will never -be compressed. This is because compression gain is pretty unlikely and -we prefer to save CPU cycles. Messages over that size are always -compressed. However, it is checked if there is a gain in compression -and only if there is, the compressed message is transmitted. Otherwise, -the uncompressed messages is transmitted. This saves the receiver CPU -cycles for decompression. It also prevents small message to actually -become larger in compressed form. -

    Please note that when a TCP transport is used, -compression will also turn on syslog-transport-tls framing. See the "o" -option for important information on the implications.

    -

    Compressed messages are automatically detected and -decompressed by the receiver. There is nothing that needs to be -configured on the receiver side.

    -
    -

    o

    -    		This option is experimental. Use at your own -risk and only if you know why you need it! If in doubt, do NOT turn it -on. -

    This option is only valid for plain TCP based -transports. It selects a different framing based on IETF internet draft -syslog-transport-tls-06. This framing offers some benefits over -traditional LF-based framing. However, the standardization effort is -not yet complete. There may be changes in upcoming versions of this -standard. Rsyslog will be kept in line with the standard. There is some -chance that upcoming changes will be incompatible to the current -specification. In this case, all systems using -transport-tls framing -must be upgraded. There will be no effort made to retain compatibility -between different versions of rsyslog. The primary reason for that is -that it seems technically impossible to provide compatibility between -some of those changes. So you should take this note very serious. It is -not something we do not *like* to do (and may change our mind if enough -people beg...), it is something we most probably *can not* do for -technical reasons (aka: you can beg as much as you like, it won't -change anything...).

    -

    The most important implication is that compressed syslog -messages via TCP must be considered with care. Unfortunately, it is -technically impossible to transfer compressed records over traditional -syslog plain tcp transports, so you are left with two evil choices...

    -
    -


    -The hostname may be followed by a colon and the destination port.

    -

    The following is an example selector line with forwarding:

    -

    *.*    @@(o,z9)192.168.0.1:1470

    -

    In this example, messages are forwarded via plain TCP with -experimental framing and maximum compression to the host 192.168.0.1 at -port 1470.

    -

    *.* @192.168.0.1

    -

    In the example above, messages are forwarded via UDP to the -machine 192.168.0.1, the destination port defaults to 514. Messages -will not be compressed.

    -

    Note that IPv6 addresses contain colons. So if an IPv6 address is specified -in the hostname part, rsyslogd could not detect where the IP address ends -and where the port starts. There is a syntax extension to support this: -put squary brackets around the address (e.g. "[2001::1]"). Square -brackets also work with real host names and IPv4 addresses, too. -

    A valid sample to send messages to the IPv6 host 2001::1 at port 515 -is as follows: -

    *.* @[2001::1]:515 -

    This works with TCP, too. -

    Note to sysklogd users: sysklogd does not -support RFC 3164 format, which is the default forwarding template in -rsyslog. As such, you will experience duplicate hostnames if rsyslog is -the sender and sysklogd is the receiver. The fix is simple: you need to -use a different template. Use that one:

    -

    $template -sysklogd,"<%PRI%>%TIMESTAMP% %syslogtag%%msg%\""
    -*.* @192.168.0.1;sysklogd

    -

    List of Users

    -

    Usually critical messages are also directed to "root'' on -that machine. You can specify a list of users that shall get the -message by simply writing the login. You may specify more than one user -by separating them with commas (",''). If they're logged in they get -the message. Don't think a mail would be sent, that might be too late.

    -

    Everyone logged on

    -

    Emergency messages often go to all users currently online to -notify them that something strange is happening with the system. To -specify this wall(1)-feature use an asterisk ("*'').

    -

    Call Plugin

    -

    This is a generic way to call an output plugin. The plugin -must support this functionality. Actual parameters depend on the -module, so see the module's doc on what to supply. The general syntax -is as follows:

    -

    :modname:params;template

    -

    Currently, the ommysql database output module supports this -syntax (in addtion to the ">" syntax it traditionally -supported). For ommysql, the module name is "ommysql" and the params -are the traditional ones. The ;template part is not module specific, it -is generic rsyslog functionality available to all modules.

    -

    As an example, the ommysql module may be called as follows:

    -

    :ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate

    -

    For details, please see the "Database Table" section of this -documentation.

    -

    Note: as of this writing, the ":modname:" part is hardcoded -into the module. So the name to use is not necessarily the name the -module's plugin file is called.

    -

    Database Table

    -

    This allows logging of the message to a database table. -Currently, only MySQL databases are supported. However, other database -drivers will most probably be developed as plugins. By default, a MonitorWare-compatible -schema is required for this to work. You can create that schema with -the createDB.SQL file that came with the rsyslog package. You can also
    -use any other schema of your liking - you just need to define a proper -template and assign this template to the action.
    -
    -The database writer is called by specifying a greater-then sign -(">") in front of the database connect information. Immediately -after that
    -sign the database host name must be given, a comma, the database name, -another comma, the database user, a comma and then the user's password. -If a specific template is to be used, a semicolon followed by the -template name can follow the connect information. This is as follows:
    -
    ->dbhost,dbname,dbuser,dbpassword;dbtemplate

    -

    Important: to use the database functionality, the -MySQL output module must be loaded in the config file BEFORE -the first database table action is used. This is done by placing the

    -

    $ModLoad ommysql

    -

    directive some place above the first use of the database write -(we recommend doing at the the beginning of the config file).

    -

    Discard

    -

    If the discard action is carried out, the received message is -immediately discarded. No further processing of it occurs. Discard has -primarily been added to filter out messages before carrying on any -further processing. For obvious reasons, the results of "discard" are -depending on where in the configuration file it is being used. Please -note that once a message has been discarded there is no way to retrieve -it in later configuration file lines.

    -

    Discard can be highly effective if you want to filter out some -annoying messages that otherwise would fill your log files. To do that, -place the discard actions early in your log files. This often plays -well with property-based filters, giving you great freedom in -specifying what you do not want.

    -

    Discard is just the single tilde character with no further -parameters:

    -

    ~

    -

    For example,

    -

    *.*   ~

    -

    discards everything (ok, you can achive the same by not -running rsyslogd at all...).

    -

    Output Channel

    -

    Binds an output channel definition (see there for details) to -this action. Output channel actions must start with a $-sign, e.g. if -you would like to bind your output channel definition "mychannel" to -the action, use "$mychannel". Output channels support template -definitions like all all other actions.

    -

    Shell Execute

    -

    This executes a program in a subshell. The program is passed -the template-generated message as the only command line parameter. -Rsyslog waits until the program terminates and only then continues to -run.

    -

    ^program-to-execute;template

    -

    The program-to-execute can be any valid executable. It -receives the template string as a single parameter (argv[1]).

    -

    WARNING: The Shell Execute action was added -to serve an urgent need. While it is considered reasonable save when -used with some thinking, its implications must be considered. The -current implementation uses a system() call to execute the command. -This is not the best way to do it (and will hopefully changed in -further releases). Also, proper escaping of special characters is done -to prevent command injection. However, attackers always find smart ways -to circumvent escaping, so we can not say if the escaping applied will -really safe you from all hassles. Lastly, rsyslog will wait until the -shell command terminates. Thus, a program error in it (e.g. an infinite -loop) can actually disable rsyslog. Even without that, during the -programs run-time no messages are processed by rsyslog. As the IP -stacks buffers are quickly overflowed, this bears an increased risk of -message loss. You must be aware of these implications. Even though they -are severe, there are several cases where the "shell execute" action is -very useful. This is the reason why we have included it in its current -form. To mitigate its risks, always a) test your program thoroughly, b) -make sure its runtime is as short as possible (if it requires a longer -run-time, you might want to spawn your own sub-shell asynchronously), -c) apply proper firewalling so that only known senders can send syslog -messages to rsyslog. Point c) is especially important: if rsyslog is -accepting message from any hosts, chances are much higher that an -attacker might try to exploit the "shell execute" action.

    -

    TEMPLATE NAME

    -

    Every ACTION can be followed by a template name. If so, that -template is used for message formatting. If no name is given, a -hard-coded default template is used for the action. There can only be -one template name for each given action. The default template is -specific to each action. For a description of what a template is and -what you can do with it, see "TEMPLATES" at the top of this document.

    -

    EXAMPLES

    -

    Below are example for templates and selector lines. I hope +

    Templates

    +

    Output Channels

    +

    Filter Conditions

    +

    Actions

    +

    Examples

    +

    Here you will find examples for templates and selector lines. I hope they are self-explanatory. If not, please see www.monitorware.com/rsyslog/ for advise.

    -

    TEMPLATES

    -

    Please note that the samples are split across multiple lines. -A template MUST NOT actually be split across multiple lines.
    -
    -A template that resembles traditional syslogd file output:
    -$template TraditionalFormat,"%timegenerated% %HOSTNAME%
    -%syslogtag%%msg:::drop-last-lf%\n"
    -
    -A template that tells you a little more about the message:
    -$template -precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,
    -%syslogtag%,%msg%\n"
    -
    -A template for RFC 3164 format:
    -$template RFC3164fmt,"<%PRI%>%TIMESTAMP% %HOSTNAME% -%syslogtag%%msg%"
    -
    -A template for the format traditonally used for user messages:
    -$template usermsg," XXXX%syslogtag%%msg%\n\r"
    -
    -And a template with the traditonal wall-message format:
    -$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at -%timegenerated%
    -
    -A template that can be used for the database write (please note the SQL
    -template option)
    -$template MySQLInsert,"insert iut, message, receivedat values
    -('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')
    -into systemevents\r\n", SQL
    -
    -The following template emulates WinSyslog -format (it's an Adiscon -format, you do not feel bad if you don't know it ;)). It's interesting -to see how it takes different parts out of the date stamps. What -happens is that the date stamp is split into the actual date and time -and the these two are combined with just a comma in between them.
    -
    -$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,
    -%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,
    -%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,
    -%syslogtag%%msg%\n"

    -

    SELECTOR LINES

    -

    # Store critical stuff in critical
    -#
    -*.=crit;kern.none /var/adm/critical
    -
    -This will store all messages with the priority crit in the file -/var/adm/critical, except for any kernel message.
    -
    -
    -# Kernel messages are first, stored in the kernel
    -# file, critical messages and higher ones also go
    -# to another host and to the console. Messages to
    -# the host finlandia are forwarded in RFC 3164
    -# format (using the template defined above).
    -#
    -kern.* /var/adm/kernel
    -kern.crit @finlandia;RFC3164fmt
    -kern.crit /dev/console
    -kern.info;kern.!err /var/adm/kernel-info
    -
    -The first rule direct any message that has the kernel facility to the -file /var/adm/kernel.
    -
    -The second statement directs all kernel messages of the priority crit -and higher to the remote host finlandia. This is useful, because if the -host crashes and the disks get irreparable errors you might not be able -to read the stored messages. If they're on a remote host, too, you -still can try to find out the reason for the crash.
    -
    -The third rule directs these messages to the actual console, so the -person who works on the machine will get them, too.
    -
    -The fourth line tells rsyslogd to save all kernel messages that come -with priorities from info up to warning in the file -/var/adm/kernel-info. Everything from err and higher is excluded.
    -
    -
    -# The tcp wrapper loggs with mail.info, we display
    -# all the connections on tty12
    -#
    -mail.=info /dev/tty12
    -
    -This directs all messages that uses mail.info (in source LOG_MAIL | -LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper -tcpd(8) uses this as it's default.
    -
    -
    -# Store all mail concerning stuff in a file
    -#
    -mail.*;mail.!=info /var/adm/mail
    -
    -This pattern matches all messages that come with the mail facility, -except for the info priority. These will be stored in the file -/var/adm/mail.
    -
    -
    -# Log all mail.info and news.info messages to info
    -#
    -mail,news.=info /var/adm/info
    -
    -This will extract all messages that come either with mail.info or with -news.info and store them in the file /var/adm/info.
    -
    -
    -# Log info and notice messages to messages file
    -#
    -*.=info;*.=notice;\
    -mail.none /var/log/messages
    -
    -This lets rsyslogd log all messages that come with either the info or -the notice facility into the file /var/log/messages, except for all
    -messages that use the mail facility.
    -
    -
    -# Log info messages to messages file
    -#
    -*.=info;\
    -mail,news.none /var/log/messages
    -
    -This statement causes rsyslogd to log all messages that come with the -info priority to the file /var/log/messages. But any message coming -either with the mail or the news facility will not be stored.
    -
    -
    -# Emergency messages will be displayed using wall
    -#
    -*.=emerg *
    -
    -This rule tells rsyslogd to write all emergency messages to all -currently logged in users. This is the wall action.
    -
    -
    -# Messages of the priority alert will be directed
    -# to the operator
    -#
    -*.alert root,rgerhards
    -
    -This rule directs all messages with a priority of alert or higher to -the terminals of the operator, i.e. of the users "root'' and -"rgerhards'' if they're logged in.
    -
    -
    -*.* @finlandia
    -
    -This rule would redirect all messages to a remote host called -finlandia. This is useful especially in a cluster of machines where all -syslog messages will be stored on only one machine.
    -
    -In the format shown above, UDP is used for transmitting the message. -The destination port is set to the default auf 514. Rsyslog is also -capable of using much more secure and reliable TCP sessions for message -forwarding. Also, the destination port can be specified. To select TCP, -simply add one additional @ in front of the host name (that is, @host -is UPD, @@host is TCP). For example:
    -
    -
    -*.* @@finlandia
    -
    -To specify the destination port on the remote machine, use a colon -followed by the port number after the machine name. The following -forwards to port 1514 on finlandia:
    -
    -
    -*.* @@finlandia:1514
    -
    -This syntax works both with TCP and UDP based syslog. However, you will -probably primarily need it for TCP, as there is no well-accepted port -for this transport (it is non-standard). For UDP, you can usually stick -with the default auf 514, but might want to modify it for security rea-
    -sons. If you would like to do that, it's quite easy:
    -
    -
    -*.* @finlandia:1514
    -
    -
    -
    -*.* >dbhost,dbname,dbuser,dbpassword;dbtemplate
    -
    -This rule writes all message to the database "dbname" hosted on -"dbhost". The login is done with user "dbuser" and password -"dbpassword". The actual table that is updated is specified within the -template (which contains the insert statement). The template is called -"dbtemplate" in this case.

    -

    :msg,contains,"error" @errorServer

    -

    This rule forwards all messages that contain the word "error" -in the msg part to the server "errorServer". Forwarding is via UDP. -Please note the colon in fron

    -

    CONFIGURATION FILE SYNTAX DIFFERENCES

    +

    Configuration File Syntax Differences

    Rsyslogd uses a slightly different syntax for its configuration file than the original BSD sources. Originally all messages of a specific priority and above were forwarded to the log @@ -1272,4 +68,15 @@ additional features (like template and database support). For obvious reasons, the syntax for defining such features is available in rsyslogd, only.

    - + +

    [back to top] +[manual index] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + + +> diff --git a/doc/rsyslog_high_database_rate.html b/doc/rsyslog_high_database_rate.html index 158a4df6..2bae58c6 100644 --- a/doc/rsyslog_high_database_rate.html +++ b/doc/rsyslog_high_database_rate.html @@ -7,6 +7,7 @@ +back

    Handling a massive syslog database insert rate with Rsyslog

    @@ -171,6 +172,14 @@ comments or find bugs (I *do* bugs - no way... ;)), please http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    diff --git a/doc/rsyslog_mysql.html b/doc/rsyslog_mysql.html index 753c86ec..a27bd59e 100644 --- a/doc/rsyslog_mysql.html +++ b/doc/rsyslog_mysql.html @@ -1,6 +1,6 @@ Writing syslog Data to MySQL - +back

    Writing syslog messages to MySQL

    @@ -259,4 +259,13 @@ document under the terms of the GNU Free Documentation License, Version with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/rsyslog_ng_comparison.html b/doc/rsyslog_ng_comparison.html index 2f383f78..8e121a8d 100644 --- a/doc/rsyslog_ng_comparison.html +++ b/doc/rsyslog_ng_comparison.html @@ -1,6 +1,7 @@ rsyslog vs. syslog-ng - a comparison +back

    rsyslog vs. syslog-ng

    Written by Rainer Gerhards (2008-05-06)

    @@ -584,4 +585,13 @@ feature sheet. I have not yet been able to fully work through it. In the mean time, you may want to read it in parallel. It is available at Balabit's site.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/rsyslog_stunnel.html b/doc/rsyslog_stunnel.html index 104a672e..f4f82cd0 100644 --- a/doc/rsyslog_stunnel.html +++ b/doc/rsyslog_stunnel.html @@ -1,5 +1,6 @@ +back SSL Encrypting syslog with stunnel

    SSL Encrypting Syslog with Stunnel

    @@ -236,5 +237,13 @@ comments or find bugs (I *do* bugs - no way... ;)), please Texts. A copy of the license can be viewed at http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsyslog_tls.html b/doc/rsyslog_tls.html index 7d156c3a..ebb08ebe 100644 --- a/doc/rsyslog_tls.html +++ b/doc/rsyslog_tls.html @@ -1,5 +1,6 @@ TLS (SSL) Encrypting syslog +back @@ -304,4 +305,13 @@ document under the terms of the GNU Free Documentation License, Version with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/syslog_protocol.html b/doc/syslog_protocol.html index 72de5c27..57eb9ffe 100644 --- a/doc/syslog_protocol.html +++ b/doc/syslog_protocol.html @@ -3,6 +3,7 @@ syslog-protocol support in rsyslog +back

    syslog-protocol support in rsyslog

    Rsyslog  provides a trial implementation of the proposed @@ -191,6 +192,14 @@ discussed ;)

    syslog-protocol should be further evaluated and be fully understood

     

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    -- cgit From 3c615c60beb851f3a42cea3fcc31f4a2243cedaa Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 7 Nov 2008 14:49:24 +0100 Subject: doc: added link to regular expression checker online tool --- doc/manual.html | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual.html b/doc/manual.html index f1b7d657..1d09d460 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -33,6 +33,7 @@ the links below for the

    • troubleshooting rsyslog problems
    • configuration file syntax (rsyslog.conf)
      • +
    • a regular expression checker/generator tool for rsyslog
    • property replacer, an important core component
    • a commented sample rsyslog.conf
    • rsyslog bug list
      • -- cgit From 170d0d6f375241e0d0ca85a1327df82165fec439 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 11 Nov 2008 11:00:37 +0100 Subject: added forgotten files they were new after restructuring the doc... --- doc/Makefile.am | 7 + doc/rsyslog_conf_actions.html | 336 ++++++++++++++++++++++++++++++++++++++++ doc/rsyslog_conf_examples.html | 209 +++++++++++++++++++++++++ doc/rsyslog_conf_filter.html | 280 +++++++++++++++++++++++++++++++++ doc/rsyslog_conf_global.html | 227 +++++++++++++++++++++++++++ doc/rsyslog_conf_modules.html | 53 +++++++ doc/rsyslog_conf_output.html | 81 ++++++++++ doc/rsyslog_conf_templates.html | 150 ++++++++++++++++++ 8 files changed, 1343 insertions(+) create mode 100644 doc/rsyslog_conf_actions.html create mode 100644 doc/rsyslog_conf_examples.html create mode 100644 doc/rsyslog_conf_filter.html create mode 100644 doc/rsyslog_conf_global.html create mode 100644 doc/rsyslog_conf_modules.html create mode 100644 doc/rsyslog_conf_output.html create mode 100644 doc/rsyslog_conf_templates.html diff --git a/doc/Makefile.am b/doc/Makefile.am index 22d368e0..fef1e44c 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -98,6 +98,13 @@ html_files = \ omrelp.html \ status.html \ troubleshoot.html \ + rsyslog_conf_actions.html \ + rsyslog_conf_examples.html \ + rsyslog_conf_filter.html \ + rsyslog_conf_global.html \ + rsyslog_conf_modules.html \ + rsyslog_conf_output.html \ + rsyslog_conf_templates.html \ src/classes.dia EXTRA_DIST = $(html_files) diff --git a/doc/rsyslog_conf_actions.html b/doc/rsyslog_conf_actions.html new file mode 100644 index 00000000..2ef3f4b0 --- /dev/null +++ b/doc/rsyslog_conf_actions.html @@ -0,0 +1,336 @@ + +Actions - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Actions

      +

      The action field of a rule describes what to do with the +message. In general, message content is written to a kind of "logfile". +But also other actions might be done, like writing to a database table +or forwarding to another host.
      +
      +Templates can be used with all actions. If used, the specified template +is used to generate the message content (instead of the default +template). To specify a template, write a semicolon after the action +value immediately followed by the template name.
      +
      +Beware: templates MUST be defined BEFORE they are used. It is OK to +define some templates, then use them in selector lines, define more +templates and use use them in the following selector lines. But it is +NOT permitted to use a template in a selector line that is above its +definition. If you do this, the action will be ignored.

      +

      You can have multiple actions for a single selector  (or +more precisely a single filter of such a selector line). Each action +must be on its own line and the line must start with an ampersand +('&') character and have no filters. An example would be

      +

      *.=crit rger
      +& root
      +& /var/log/critmsgs

      +

      These three lines send critical messages to the user rger and +root and also store them in /var/log/critmsgs. Using multiple +actions per selector is convenient and also offers +a performance benefit. As the filter needs to be evaluated +only once, there is less computation required to process the directive +compared to the otherwise-equal config directives below:

      +

      *.=crit rger
      +*.=crit root
      +*.=crit /var/log/critmsgs

      +

       

      +

      Regular File

      +

      Typically messages are logged to real files. The file has to +be specified with full pathname, beginning with a slash "/''.
      +
      +
      +You may prefix each entry with the minus "-'' sign to omit syncing the +file after every logging. Note that you might lose information if the +system crashes right behind a write attempt. Nevertheless this might +give you back some performance, especially if you run programs that use +logging in a very verbose manner.

      +

      If your system is connected to a reliable UPS and you receive +lots of log data (e.g. firewall logs), it might be a very good idea to +turn of +syncing by specifying the "-" in front of the file name.

      +

      The filename can be either static (always +the same) or dynamic (different based on message +received). The later is useful if you would automatically split +messages into different files based on some message criteria. For +example, dynamic file name selectors allow you to split messages into +different files based on the host that sent them. With dynamic file +names, everything is automatic and you do not need any filters.

      +

      It works via the template system. First, you define a template +for the file name. An example can be seen above in the description of +template. We will use the "DynFile" template defined there. Dynamic +filenames are indicated by specifying a questions mark "?" instead of a +slash, followed by the template name. Thus, the selector line for our +dynamic file name would look as follows:

      +
      +*.* ?DynFile +
      +

      That's all you need to do. Rsyslog will now automatically +generate file names for you and store the right messages into the right +files. Please note that the minus sign also works with dynamic file +name selectors. Thus, to avoid syncing, you may use

      +
      +*.* -?DynFile
      +

      And of course you can use templates to specify the output +format:

      +
      +*.* ?DynFile;MyTemplate
      +

      A word of caution: rsyslog creates files as +needed. So if a new host is using your syslog server, rsyslog will +automatically create a new file for it.

      +

      Creating directories is also supported. For +example you can use the hostname as directory and the program name as +file name:

      +
      +$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"
      +

      Named Pipes

      +

      This version of rsyslogd(8) has support for logging output to +named pipes (fifos). A fifo or named pipe can be used as a destination +for log messages by prepending a pipe symbol ("|'') to the name of the +file. This is handy for debugging. Note that the fifo must be created +with the mkfifo(1) command before rsyslogd(8) is started.

      +

      Terminal and Console

      +

      If the file you specified is a tty, special tty-handling is +done, same with /dev/console.

      +

      Remote Machine

      +

      Rsyslogd provides full remote logging, i.e. is able to send +messages to a remote host running rsyslogd(8) and to receive messages +from remote hosts. Using this feature you're able to control all syslog +messages on one host, if all other machines will log remotely to that. +This tears down
      +administration needs.
      +
      +Please note that this version of rsyslogd by default does NOT +forward messages it has received from the network to another host. +Specify the "-h" option to enable this.

      +

      To forward messages to another host, prepend the hostname with +the at sign ("@"). A single at sign means that messages will +be forwarded via UDP protocol (the standard for syslog). If you prepend +two at signs ("@@"), the messages will be transmitted via TCP. Please +note that plain TCP based syslog is not officially standardized, but +most major syslogds support it (e.g. syslog-ng or WinSyslog). The +forwarding action indicator (at-sign) can be followed by one or more +options. If they are given, they must be immediately (without a space) +following the final at sign and be enclosed in parenthesis. The +individual options must be separated by commas. The following options +are right now defined:

      + + + + + + + + + + + +
      +

      z<number>

      +      		Enable zlib-compression for the message. The +<number> is the compression level. It can be 1 (lowest +gain, lowest CPU overhead) to 9 (maximum compression, highest CPU +overhead). The level can also be 0, which means "no compression". If +given, the "z" option is ignored. So this does not make an awful lot of +sense. There is hardly a difference between level 1 and 9 for typical +syslog messages. You can expect a compression gain between 0% and 30% +for typical messages. Very chatty messages may compress up to 50%, but +this is seldom seen with typically traffic. Please note that rsyslogd +checks the compression gain. Messages with 60 bytes or less will never +be compressed. This is because compression gain is pretty unlikely and +we prefer to save CPU cycles. Messages over that size are always +compressed. However, it is checked if there is a gain in compression +and only if there is, the compressed message is transmitted. Otherwise, +the uncompressed messages is transmitted. This saves the receiver CPU +cycles for decompression. It also prevents small message to actually +become larger in compressed form. +

      Please note that when a TCP transport is used, +compression will also turn on syslog-transport-tls framing. See the "o" +option for important information on the implications.

      +

      Compressed messages are automatically detected and +decompressed by the receiver. There is nothing that needs to be +configured on the receiver side.

      +
      +

      o

      +      		This option is experimental. Use at your own +risk and only if you know why you need it! If in doubt, do NOT turn it +on. +

      This option is only valid for plain TCP based +transports. It selects a different framing based on IETF internet draft +syslog-transport-tls-06. This framing offers some benefits over +traditional LF-based framing. However, the standardization effort is +not yet complete. There may be changes in upcoming versions of this +standard. Rsyslog will be kept in line with the standard. There is some +chance that upcoming changes will be incompatible to the current +specification. In this case, all systems using -transport-tls framing +must be upgraded. There will be no effort made to retain compatibility +between different versions of rsyslog. The primary reason for that is +that it seems technically impossible to provide compatibility between +some of those changes. So you should take this note very serious. It is +not something we do not *like* to do (and may change our mind if enough +people beg...), it is something we most probably *can not* do for +technical reasons (aka: you can beg as much as you like, it won't +change anything...).

      +

      The most important implication is that compressed syslog +messages via TCP must be considered with care. Unfortunately, it is +technically impossible to transfer compressed records over traditional +syslog plain tcp transports, so you are left with two evil choices...

      +
      +


      +The hostname may be followed by a colon and the destination port.

      +

      The following is an example selector line with forwarding:

      +

      *.*    @@(o,z9)192.168.0.1:1470

      +

      In this example, messages are forwarded via plain TCP with +experimental framing and maximum compression to the host 192.168.0.1 at +port 1470.

      +

      *.* @192.168.0.1

      +

      In the example above, messages are forwarded via UDP to the +machine 192.168.0.1, the destination port defaults to 514. Messages +will not be compressed.

      +

      Note that IPv6 addresses contain colons. So if an IPv6 address is specified +in the hostname part, rsyslogd could not detect where the IP address ends +and where the port starts. There is a syntax extension to support this: +put squary brackets around the address (e.g. "[2001::1]"). Square +brackets also work with real host names and IPv4 addresses, too. +

      A valid sample to send messages to the IPv6 host 2001::1 at port 515 +is as follows: +

      *.* @[2001::1]:515 +

      This works with TCP, too. +

      Note to sysklogd users: sysklogd does not +support RFC 3164 format, which is the default forwarding template in +rsyslog. As such, you will experience duplicate hostnames if rsyslog is +the sender and sysklogd is the receiver. The fix is simple: you need to +use a different template. Use that one:

      +

      $template +sysklogd,"<%PRI%>%TIMESTAMP% %syslogtag%%msg%\""
      +*.* @192.168.0.1;sysklogd

      +

      List of Users

      +

      Usually critical messages are also directed to "root'' on +that machine. You can specify a list of users that shall get the +message by simply writing the login. You may specify more than one user +by separating them with commas (",''). If they're logged in they get +the message. Don't think a mail would be sent, that might be too late.

      +

      Everyone logged on

      +

      Emergency messages often go to all users currently online to +notify them that something strange is happening with the system. To +specify this wall(1)-feature use an asterisk ("*'').

      +

      Call Plugin

      +

      This is a generic way to call an output plugin. The plugin +must support this functionality. Actual parameters depend on the +module, so see the module's doc on what to supply. The general syntax +is as follows:

      +

      :modname:params;template

      +

      Currently, the ommysql database output module supports this +syntax (in addtion to the ">" syntax it traditionally +supported). For ommysql, the module name is "ommysql" and the params +are the traditional ones. The ;template part is not module specific, it +is generic rsyslog functionality available to all modules.

      +

      As an example, the ommysql module may be called as follows:

      +

      :ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate

      +

      For details, please see the "Database Table" section of this +documentation.

      +

      Note: as of this writing, the ":modname:" part is hardcoded +into the module. So the name to use is not necessarily the name the +module's plugin file is called.

      +

      Database Table

      +

      This allows logging of the message to a database table. +Currently, only MySQL databases are supported. However, other database +drivers will most probably be developed as plugins. By default, a MonitorWare-compatible +schema is required for this to work. You can create that schema with +the createDB.SQL file that came with the rsyslog package. You can also
      +use any other schema of your liking - you just need to define a proper +template and assign this template to the action.
      +
      +The database writer is called by specifying a greater-then sign +(">") in front of the database connect information. Immediately +after that
      +sign the database host name must be given, a comma, the database name, +another comma, the database user, a comma and then the user's password. +If a specific template is to be used, a semicolon followed by the +template name can follow the connect information. This is as follows:
      +
      +>dbhost,dbname,dbuser,dbpassword;dbtemplate

      +

      Important: to use the database functionality, the +MySQL output module must be loaded in the config file BEFORE +the first database table action is used. This is done by placing the

      +

      $ModLoad ommysql

      +

      directive some place above the first use of the database write +(we recommend doing at the the beginning of the config file).

      +

      Discard

      +

      If the discard action is carried out, the received message is +immediately discarded. No further processing of it occurs. Discard has +primarily been added to filter out messages before carrying on any +further processing. For obvious reasons, the results of "discard" are +depending on where in the configuration file it is being used. Please +note that once a message has been discarded there is no way to retrieve +it in later configuration file lines.

      +

      Discard can be highly effective if you want to filter out some +annoying messages that otherwise would fill your log files. To do that, +place the discard actions early in your log files. This often plays +well with property-based filters, giving you great freedom in +specifying what you do not want.

      +

      Discard is just the single tilde character with no further +parameters:

      +

      ~

      +

      For example,

      +

      *.*   ~

      +

      discards everything (ok, you can achive the same by not +running rsyslogd at all...).

      +

      Output Channel

      +

      Binds an output channel definition (see there for details) to +this action. Output channel actions must start with a $-sign, e.g. if +you would like to bind your output channel definition "mychannel" to +the action, use "$mychannel". Output channels support template +definitions like all all other actions.

      +

      Shell Execute

      +

      This executes a program in a subshell. The program is passed +the template-generated message as the only command line parameter. +Rsyslog waits until the program terminates and only then continues to +run.

      +

      ^program-to-execute;template

      +

      The program-to-execute can be any valid executable. It +receives the template string as a single parameter (argv[1]).

      +

      WARNING: The Shell Execute action was added +to serve an urgent need. While it is considered reasonable save when +used with some thinking, its implications must be considered. The +current implementation uses a system() call to execute the command. +This is not the best way to do it (and will hopefully changed in +further releases). Also, proper escaping of special characters is done +to prevent command injection. However, attackers always find smart ways +to circumvent escaping, so we can not say if the escaping applied will +really safe you from all hassles. Lastly, rsyslog will wait until the +shell command terminates. Thus, a program error in it (e.g. an infinite +loop) can actually disable rsyslog. Even without that, during the +programs run-time no messages are processed by rsyslog. As the IP +stacks buffers are quickly overflowed, this bears an increased risk of +message loss. You must be aware of these implications. Even though they +are severe, there are several cases where the "shell execute" action is +very useful. This is the reason why we have included it in its current +form. To mitigate its risks, always a) test your program thoroughly, b) +make sure its runtime is as short as possible (if it requires a longer +run-time, you might want to spawn your own sub-shell asynchronously), +c) apply proper firewalling so that only known senders can send syslog +messages to rsyslog. Point c) is especially important: if rsyslog is +accepting message from any hosts, chances are much higher that an +attacker might try to exploit the "shell execute" action.

      +

      Template Name

      +

      Every ACTION can be followed by a template name. If so, that +template is used for message formatting. If no name is given, a +hard-coded default template is used for the action. There can only be +one template name for each given action. The default template is +specific to each action. For a description of what a template is and +what you can do with it, see "TEMPLATES" at the top of this document.

      + + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_examples.html b/doc/rsyslog_conf_examples.html new file mode 100644 index 00000000..b46460e5 --- /dev/null +++ b/doc/rsyslog_conf_examples.html @@ -0,0 +1,209 @@ + +Examples - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Examples

      +

      Below are example for templates and selector lines. I hope +they are self-explanatory. If not, please see +www.monitorware.com/rsyslog/ for advise.

      +

      TEMPLATES

      +

      Please note that the samples are split across multiple lines. +A template MUST NOT actually be split across multiple lines.
      +
      +A template that resembles traditional syslogd file output:
      +$template TraditionalFormat,"%timegenerated% %HOSTNAME%
      +%syslogtag%%msg:::drop-last-lf%\n"
      +
      +A template that tells you a little more about the message:
      +$template +precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,
      +%syslogtag%,%msg%\n"
      +
      +A template for RFC 3164 format:
      +$template RFC3164fmt,"<%PRI%>%TIMESTAMP% %HOSTNAME% +%syslogtag%%msg%"
      +
      +A template for the format traditonally used for user messages:
      +$template usermsg," XXXX%syslogtag%%msg%\n\r"
      +
      +And a template with the traditonal wall-message format:
      +$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at +%timegenerated%
      +
      +A template that can be used for the database write (please note the SQL
      +template option)
      +$template MySQLInsert,"insert iut, message, receivedat values
      +('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')
      +into systemevents\r\n", SQL
      +
      +The following template emulates WinSyslog +format (it's an Adiscon +format, you do not feel bad if you don't know it ;)). It's interesting +to see how it takes different parts out of the date stamps. What +happens is that the date stamp is split into the actual date and time +and the these two are combined with just a comma in between them.
      +
      +$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,
      +%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,
      +%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,
      +%syslogtag%%msg%\n"

      +

      SELECTOR LINES

      +

      # Store critical stuff in critical
      +#
      +*.=crit;kern.none /var/adm/critical
      +
      +This will store all messages with the priority crit in the file +/var/adm/critical, except for any kernel message.
      +
      +
      +# Kernel messages are first, stored in the kernel
      +# file, critical messages and higher ones also go
      +# to another host and to the console. Messages to
      +# the host finlandia are forwarded in RFC 3164
      +# format (using the template defined above).
      +#
      +kern.* /var/adm/kernel
      +kern.crit @finlandia;RFC3164fmt
      +kern.crit /dev/console
      +kern.info;kern.!err /var/adm/kernel-info
      +
      +The first rule direct any message that has the kernel facility to the +file /var/adm/kernel.
      +
      +The second statement directs all kernel messages of the priority crit +and higher to the remote host finlandia. This is useful, because if the +host crashes and the disks get irreparable errors you might not be able +to read the stored messages. If they're on a remote host, too, you +still can try to find out the reason for the crash.
      +
      +The third rule directs these messages to the actual console, so the +person who works on the machine will get them, too.
      +
      +The fourth line tells rsyslogd to save all kernel messages that come +with priorities from info up to warning in the file +/var/adm/kernel-info. Everything from err and higher is excluded.
      +
      +
      +# The tcp wrapper loggs with mail.info, we display
      +# all the connections on tty12
      +#
      +mail.=info /dev/tty12
      +
      +This directs all messages that uses mail.info (in source LOG_MAIL | +LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper +tcpd(8) uses this as it's default.
      +
      +
      +# Store all mail concerning stuff in a file
      +#
      +mail.*;mail.!=info /var/adm/mail
      +
      +This pattern matches all messages that come with the mail facility, +except for the info priority. These will be stored in the file +/var/adm/mail.
      +
      +
      +# Log all mail.info and news.info messages to info
      +#
      +mail,news.=info /var/adm/info
      +
      +This will extract all messages that come either with mail.info or with +news.info and store them in the file /var/adm/info.
      +
      +
      +# Log info and notice messages to messages file
      +#
      +*.=info;*.=notice;\
      +mail.none /var/log/messages
      +
      +This lets rsyslogd log all messages that come with either the info or +the notice facility into the file /var/log/messages, except for all
      +messages that use the mail facility.
      +
      +
      +# Log info messages to messages file
      +#
      +*.=info;\
      +mail,news.none /var/log/messages
      +
      +This statement causes rsyslogd to log all messages that come with the +info priority to the file /var/log/messages. But any message coming +either with the mail or the news facility will not be stored.
      +
      +
      +# Emergency messages will be displayed using wall
      +#
      +*.=emerg *
      +
      +This rule tells rsyslogd to write all emergency messages to all +currently logged in users. This is the wall action.
      +
      +
      +# Messages of the priority alert will be directed
      +# to the operator
      +#
      +*.alert root,rgerhards
      +
      +This rule directs all messages with a priority of alert or higher to +the terminals of the operator, i.e. of the users "root'' and +"rgerhards'' if they're logged in.
      +
      +
      +*.* @finlandia
      +
      +This rule would redirect all messages to a remote host called +finlandia. This is useful especially in a cluster of machines where all +syslog messages will be stored on only one machine.
      +
      +In the format shown above, UDP is used for transmitting the message. +The destination port is set to the default auf 514. Rsyslog is also +capable of using much more secure and reliable TCP sessions for message +forwarding. Also, the destination port can be specified. To select TCP, +simply add one additional @ in front of the host name (that is, @host +is UPD, @@host is TCP). For example:
      +
      +
      +*.* @@finlandia
      +
      +To specify the destination port on the remote machine, use a colon +followed by the port number after the machine name. The following +forwards to port 1514 on finlandia:
      +
      +
      +*.* @@finlandia:1514
      +
      +This syntax works both with TCP and UDP based syslog. However, you will +probably primarily need it for TCP, as there is no well-accepted port +for this transport (it is non-standard). For UDP, you can usually stick +with the default auf 514, but might want to modify it for security rea-
      +sons. If you would like to do that, it's quite easy:
      +
      +
      +*.* @finlandia:1514
      +
      +
      +
      +*.* >dbhost,dbname,dbuser,dbpassword;dbtemplate
      +
      +This rule writes all message to the database "dbname" hosted on +"dbhost". The login is done with user "dbuser" and password +"dbpassword". The actual table that is updated is specified within the +template (which contains the insert statement). The template is called +"dbtemplate" in this case.

      +

      :msg,contains,"error" @errorServer

      +

      This rule forwards all messages that contain the word "error" +in the msg part to the server "errorServer". Forwarding is via UDP. +Please note the colon in fron

      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html new file mode 100644 index 00000000..55244c15 --- /dev/null +++ b/doc/rsyslog_conf_filter.html @@ -0,0 +1,280 @@ + +Filter Conditions - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Filter Conditions

      +

      Rsyslog offers four different types "filter conditions":

      +
        +
      • BSD-style blocks
        • +
      • "traditional" severity and facility based selectors
        • +
      • property-based filters
        • +
      • expression-based filters
        • +
      +

      Blocks

      +

      Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each +block of lines is separated from the previous block by a program or +hostname specification. A block will only log messages corresponding to +the most recent program and hostname specifications given. Thus, a +block which selects ‘ppp’ as the program, directly followed by a block +that selects messages from the hostname ‘dialhost’, then the second +block will only log messages from the ppp program on dialhost. +

      +

      A program specification is a line beginning with ‘!prog’ and +the following blocks will be associated with calls to syslog from that +specific program. A program specification for ‘foo’ will also match any +message logged by the kernel with the prefix ‘foo: ’. Alternatively, a +program specification ‘-foo’ causes the following blocks to be applied +to messages from any program but the one specified. A hostname +specification of the form ‘+hostname’ and the following blocks will be +applied to messages received from the specified hostname. +Alternatively, a hostname specification ‘-hostname’ causes the +following blocks to be applied to messages from any host but the one +specified. If the hostname is given as ‘@’, the local hostname will be +used. (NOT YET IMPLEMENTED) A program or hostname specification may be +reset by giving the program or hostname as ‘*’.

      +

      Please note that the "#!prog", "#+hostname" and "#-hostname" +syntax available in BSD syslogd is not supported by rsyslogd. By +default, no hostname or program is set.

      +

      Selectors

      +

      Selectors are the traditional way of filtering syslog +messages. They have been kept in rsyslog with their original +syntax, because it is well-known, highly effective and also needed for +compatibility with stock syslogd configuration files. If you just need +to filter based on priority and facility, you should do this with +selector lines. They are not second-class citizens +in rsyslog and offer the best performance for this job.

      +

      The selector field itself again consists of two parts, a +facility and a priority, separated by a period (".''). Both parts are +case insensitive and can also be specified as decimal numbers, but +don't do that, you have been warned. Both facilities and priorities are +described in rsyslog(3). The names mentioned below correspond to the +similar LOG_-values in /usr/include/rsyslog.h.
      +
      +The facility is one of the following keywords: auth, authpriv, cron, +daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, +user, uucp and local0 through local7. The keyword security should not +be used anymore and mark is only for internal use and therefore should +not be used in applications. Anyway, you may want to specify and +redirect these messages here. The facility specifies the subsystem that +produced the message, i.e. all mail programs log with the mail facility +(LOG_MAIL) if they log using syslog.
      +
      +The priority is one of the following keywords, in ascending order: +debug, info, notice, warning, warn (same as warning), err, error (same +as err), crit, alert, emerg, panic (same as emerg). The keywords error, +warn and panic are deprecated and should not be used anymore. The +priority defines the severity of the message.
      +
      +The behavior of the original BSD syslogd is that all messages of the +specified priority and higher are logged according to the given action. +Rsyslogd behaves the same, but has some extensions.
      +
      +In addition to the above mentioned names the rsyslogd(8) understands +the following extensions: An asterisk ("*'') stands for all facilities +or all priorities, depending on where it is used (before or after the +period). The keyword none stands for no priority of the given facility.
      +
      +You can specify multiple facilities with the same priority pattern in +one statement using the comma (",'') operator. You may specify as much +facilities as you want. Remember that only the facility part from such +a statement is taken, a priority part would be skipped.

      +

      Multiple selectors may be specified for a single action using +the semicolon (";'') separator. Remember that each selector in the +selector field is capable to overwrite the preceding ones. Using this +behavior you can exclude some priorities from the pattern.

      +

      Rsyslogd has a syntax extension to the original BSD source, +that makes its use more intuitively. You may precede every priority +with an equation sign ("='') to specify only this single priority and +not any of the above. You may also (both is valid, too) precede the +priority with an exclamation mark ("!'') to ignore all that +priorities, either exact this one or this and any higher priority. If +you use both extensions than the exclamation mark must occur before the +equation sign, just use it intuitively.

      +

      Property-Based Filters

      +

      Property-based filters are unique to rsyslogd. They allow to +filter on any property, like HOSTNAME, syslogtag and msg. A list of all +currently-supported properties can be found in the property replacer documentation +(but keep in mind that only the properties, not the replacer is +supported). With this filter, each properties can be checked against a +specified value, using a specified compare operation. Currently, there +is only a single compare operation (contains) available, but additional +operations will be added in the future.

      +

      A property-based filter must start with a colon in column 0. +This tells rsyslogd that it is the new filter type. The colon must be +followed by the property name, a comma, the name of the compare +operation to carry out, another comma and then the value to compare +against. This value must be quoted. There can be spaces and tabs +between the commas. Property names and compare operations are +case-sensitive, so "msg" works, while "MSG" is an invalid property +name. In brief, the syntax is as follows:

      +

      :property, [!]compare-operation, "value"

      +

      The following compare-operations are +currently supported:

      + + + + + + + + + + + + + + + + + + + +
      containsChecks if the string provided in value is contained in +the property. There must be an exact match, wildcards are not supported.
      isequalCompares the "value" string provided and the property +contents. These two values must be exactly equal to match. The +difference to contains is that contains searches for the value anywhere +inside the property value, whereas all characters must be identical for +isequal. As such, isequal is most useful for fields like syslogtag or +FROMHOST, where you probably know the exact contents.
      startswithChecks if the value is found exactly at the beginning +of the property value. For example, if you search for "val" with +

      :msg, startswith, "val"

      +

      it will be a match if msg contains "values are in this +message" but it won't match if the msg contains "There are values in +this message" (in the later case, contains would match). Please note +that "startswith" is by far faster than regular expressions. So even +once they are implemented, it can make very much sense +(performance-wise) to use "startswith".

      +
      regexCompares the property against the provided POSIX +regular +expression.
      +

      You can use the bang-character (!) immediately in front of a +compare-operation, the outcome of this operation is negated. For +example, if msg contains "This is an informative message", the +following sample would not match:

      +

      :msg, contains, "error"

      +

      but this one matches:

      +

      :msg, !contains, "error"

      +

      Using negation can be useful if you would like to do some +generic processing but exclude some specific events. You can use the +discard action in conjunction with that. A sample would be:

      +

      *.* +/var/log/allmsgs-including-informational.log
      +:msg, contains, "informational"  ~ +
      +*.* /var/log/allmsgs-but-informational.log

      +

      Do not overlook the red tilde in line 2! In this sample, all +messages are written to the file allmsgs-including-informational.log. +Then, all messages containing the string "informational" are discarded. +That means the config file lines below the "discard line" (number 2 in +our sample) will not be applied to this message. Then, all remaining +lines will also be written to the file allmsgs-but-informational.log.

      +

      Value is a quoted string. It supports some +escape sequences:

      +

      \" - the quote character (e.g. "String with \"Quotes\"")
      +\\ - the backslash character (e.g. "C:\\tmp")

      +

      Escape sequences always start with a backslash. Additional +escape sequences might be added in the future. Backslash characters must +be escaped. Any other sequence then those outlined above is invalid and +may lead to unpredictable results.

      +

      Probably, "msg" is the most prominent use case of property +based filters. It is the actual message text. If you would like to +filter based on some message content (e.g. the presence of a specific +code), this can be done easily by:

      +

      :msg, contains, "ID-4711"

      +

      This filter will match when the message contains the string +"ID-4711". Please note that the comparison is case-sensitive, so it +would not match if "id-4711" would be contained in the message.

      +

      :msg, regex, "fatal .* error"

      +

      This filter uses a POSIX regular expression. It matches when +the +string contains the words "fatal" and "error" with anything in between +(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as +two spaces are required by the regular expression!).

      +

      Getting property-based filters right can sometimes be +challenging. In order to help you do it with as minimal effort as +possible, rsyslogd spits out debug information for all property-based +filters during their evaluation. To enable this, run rsyslogd in +foreground and specify the "-d" option.

      +

      Boolean operations inside property based filters (like +'message contains "ID17" or message contains "ID18"') are currently not +supported (except for "not" as outlined above). Please note that while +it is possible to query facility and severity via property-based +filters, it is far more advisable to use classic selectors (see above) +for those cases.

      +

      Expression-Based Filters

      +Expression based filters allow +filtering on arbitrary complex expressions, which can include boolean, +arithmetic and string operations. Expression filters will evolve into a +full configuration scripting language. Unfortunately, their syntax will +slightly change during that process. So if you use them now, you need +to be prepared to change your configuration files some time later. +However, we try to implement the scripting facility as soon as possible +(also in respect to stage work needed). So the window of exposure is +probably not too long.
      +
      +Expression based filters are indicated by the keyword "if" in column 1 +of a new line. They have this format:
      +
      +if expr then action-part-of-selector-line
      +
      +"If" and "then" are fixed keywords that mus be present. "expr" is a +(potentially quite complex) expression. So the expression documentation for +details. "action-part-of-selector-line" is an action, just as you know +it (e.g. "/var/log/logfile" to write to that file).
      +
      +A few quick samples:
      +
      + +*.* /var/log/file1 # the traditional way
      +if $msg contains 'error' /var/log/errlog # the expression-based way
      +       +
      +Right now, you need to specify numerical values if you would like to +check for facilities and severity. These can be found in RFC 3164. +If you don't like that, you can of course also use the textual property +- just be sure to use the right one. As expression support is enhanced, +this will change. For example, if you would like to filter on message +that have facility local0, start with "DEVNAME" and have either +"error1" or "error0" in their message content, you could use the +following filter:
      +
      + +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog
      +       +
      +Please note that the above must +all be on one line! And if you would like to store all +messages except those that contain "error1" or "error0", you just need +to add a "not":
      +
      + +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and not +($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog
      +       +
      +If you would like to do case-insensitive comparisons, use +"contains_i" instead of "contains" and "startswith_i" instead of +"startswith".
      +
      +Note that regular expressions are currently NOT +supported in expression-based filters. These will be added later when +function support is added to the expression engine (the reason is that +regular expressions will be a separate loadable module, which requires +some more prequisites before it can be implemented).
      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html new file mode 100644 index 00000000..bc618dd0 --- /dev/null +++ b/doc/rsyslog_conf_global.html @@ -0,0 +1,227 @@ + +Global Directives - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Global Directives

      +

      All global directives need to be specified on a line by their +own and must start with a dollar-sign. Here is a list in alphabetical +order. Follow links for a description.

      +

      Please note that not all directives here are actually global. Some affect +only the next action. This documentation will be changed soon. +

      Not all directives have an in-depth description right now. +Default values for them are in bold. A more in-depth description will +appear as implementation progresses. +

      +

      Be sure to read information about queues in rsyslog - +many parameter settings modify queue parameters. If in doubt, use the +default, it is usually well-chosen and applicable in most cases.

      +
        +
      • $ActionExecOnlyWhenPreviousIsSuspended
        • +
      • $ActionExecOnlyOnceEveryInterval <seconds> - +execute action only if the last execute is at last +<seconds> seconds in the past (more info in ommail, +but may be used with any action)
        • +
      • $ActionExecOnlyEveryNthTime <number> - If configured, the next action will +only be executed every n-th time. For example, if configured to 3, the first two messages +that go into the action will be dropped, the 3rd will actually cause the action to execute, +the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note: +this setting is automatically re-set when the actual action is defined.
        • +
      • $ActionExecOnlyEveryNthTimeTimeout <number-of-seconds> - has a meaning only if +$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout +setting specifies after which period the counting of "previous actions" expires and +a new action count is begun. Specify 0 (the default) to disable timeouts. +
        +Why is this option needed? Consider this case: a message comes in at, eg., 10am. That's +count 1. Then, nothing happens for the next 10 hours. At 8pm, the next +one occurs. That's count 2. Another 5 hours later, the next message +occurs, bringing the total count to 3. Thus, this message now triggers +the rule. +
        +The question is if this is desired behavior? Or should the rule only be +triggered if the messages occur within an e.g. 20 minute window? If the +later is the case, you need a +
        +$ActionExecOnlyEveryNthTimeTimeout 1200 +
        +This directive will timeout previous messages seen if they are older +than 20 minutes. In the example above, the count would now be always 1 +and consequently no rule would ever be triggered. + +
      • $ActionFileDefaultTemplate [templateName] - sets a new default template for file actions
        • +
      • $ActionFileEnableSync [on/off] - enables file +syncing capability of omfile
        • +
      • $ActionForwardDefaultTemplate [templateName] - sets a new +default template for UDP and plain TCP forwarding action
        • +
      • $ActionGSSForwardDefaultTemplate [templateName] - sets a +new default template for GSS-API forwarding action
        • +
      • $ActionQueueCheckpointInterval <number>
        • +
      • $ActionQueueDequeueSlowdown <number> [number +is timeout in microseconds (1000000us is 1sec!), +default 0 (no delay). Simple rate-limiting!]
        • +
      • $ActionQueueDiscardMark <number> [default +9750]
        • +
      • $ActionQueueDiscardSeverity <number> +[*numerical* severity! default 4 (warning)]
        • +
      • $ActionQueueFileName <name>
        • +
      • $ActionQueueHighWaterMark <number> [default +8000]
        • +
      • $ActionQueueImmediateShutdown [on/off]
        • +
      • $ActionQueueSize <number>
        • +
      • $ActionQueueLowWaterMark <number> [default +2000]
        • +
      • $ActionQueueMaxFileSize <size_nbr>, default 1m
        • +
      • $ActionQueueTimeoutActionCompletion <number> +[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means +immediate!]
        • +
      • $ActionQueueTimeoutEnqueue <number> [number +is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
        • +
      • $ActionQueueTimeoutShutdown <number> [number +is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
        • +
      • $ActionQueueWorkerTimeoutThreadShutdown +<number> [number is timeout in ms (1000ms is 1sec!), +default 60000 (1 minute)]
        • +
      • $ActionQueueType [FixedArray/LinkedList/Direct/Disk]
        • +
      • $ActionQueueSaveOnShutdown  [on/off] +
        • +
      • $ActionQueueWorkerThreads <number>, num worker threads, default 1, recommended 1
        • +
      • $ActionQueueWorkerThreadMinumumMessages <number>, default 100
        • +
      • $ActionResumeInterval
        • +
      • $ActionResumeRetryCount <number> [default 0, -1 means eternal]
        • +
      • $ActionSendResendLastMsgOnReconn <[on/off]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. +
      • $ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action +
      • $ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver +(driver-specific)
      • $ActionSendStreamDriverAuthMode <mode>,  authentication mode to use with the stream driver +(driver-specific)
      • $ActionSendStreamDriverPermittedPeer <ID>,  accepted fingerprint (SHA1) or name of remote peer +(driver-specific) - directive may go away!
        • +
      • $AllowedSender
        • +
      • $ControlCharacterEscapePrefix
        • +
      • $DebugPrintCFSyslineHandlerList
        • + +
      • $DebugPrintModuleList
        • +
      • $DebugPrintTemplateList
        • +
      • $DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
        • +
      • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
        • +
      • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
        • +
      • $DirCreateMode
        • +
      • $DirGroup
        • +
      • $DirOwner
        • +
      • $DropMsgsWithMaliciousDnsPTRRecords
        • +
      • $DropTrailingLFOnReception
        • +
      • $DynaFileCacheSize
        • +
      • $EscapeControlCharactersOnReceive
        • +
      • $ErrorMessagesToStderr [on|off] - direct rsyslogd error message to stderr (in addition to other targets)
        • +
      • $FailOnChownFailure
        • +
      • $FileCreateMode
        • +
      • $FileGroup
        • +
      • $FileOwner
        • +
      • $GssForwardServiceName
        • +
      • $GssListenServiceName
        • +
      • $GssMode
        • +
      • $HUPisRestart [on/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending +on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is +not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for +compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually +the only one that is needed e.g. for log rotation. It is recommended to set the setting to "off".
        • +
      • $IncludeConfig
      • MainMsgQueueCheckpointInterval <number>
        • +
      • $MainMsgQueueDequeueSlowdown <number> [number +is timeout in microseconds (1000000us is 1sec!), +default 0 (no delay). Simple rate-limiting!]
        • +
      • $MainMsgQueueDiscardMark <number> [default 9750]
        • +
      • $MainMsgQueueDiscardSeverity <severity> +[either a textual or numerical severity! default 4 (warning)]
        • +
      • $MainMsgQueueFileName <name>
        • +
      • $MainMsgQueueHighWaterMark <number> [default +8000]
        • +
      • $MainMsgQueueImmediateShutdown [on/off]
        • +
      • $MainMsgQueueSize
        • +
      • $MainMsgQueueLowWaterMark <number> [default +2000]
        • +
      • $MainMsgQueueMaxFileSize <size_nbr>, default +1m
        • +
      • $MainMsgQueueTimeoutActionCompletion +<number> [number is timeout in ms (1000ms is 1sec!), +default +1000, 0 means immediate!]
        • +
      • $MainMsgQueueTimeoutEnqueue <number> [number +is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
        • +
      • $MainMsgQueueTimeoutShutdown <number> [number +is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
        • +
      • $MainMsgQueueWorkerTimeoutThreadShutdown +<number> [number is timeout in ms (1000ms is 1sec!), +default 60000 (1 minute)]
        • +
      • $MainMsgQueueType [FixedArray/LinkedList/Direct/Disk]
        • +
      • $MainMsgQueueSaveOnShutdown  [on/off] +
        • +
      • $MainMsgQueueWorkerThreads <number>, num +worker threads, default 1, recommended 1
        • +
      • $MainMsgQueueWorkerThreadMinumumMessages <number>, default 100
        • +
      • $MarkMessagePeriod (immark)
        • +
      • $MaxMessageSize <size_nbr>, default 2k - allows to specify maximum supported message size +(both for sending and receiving). The default +should be sufficient for almost all cases. Do not set this below 1k, as it would cause +interoperability problems with other syslog implementations.
        +Change the setting to e.g. 32768 if you would like to +support large message sizes for IHE (32k is the current maximum +needed for IHE). I was initially tempted to set the default to 32k, +but there is a some memory footprint with the current +implementation in rsyslog. +
        If you intend to receive Windows Event Log data (e.g. via +EventReporter), you might want to +increase this number to an even higher value, as event +log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea). +Note: testing showed that 4k seems to be +the typical maximum for UDP based syslog. This is an IP stack +restriction. Not always ... but very often. If you go beyond +that value, be sure to test that rsyslogd actually does what +you think it should do ;) It is highly suggested to use a TCP based transport +instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions. +
        Note that 2k, the current default, is the smallest size that must be +supported in order to be compliant to the upcoming new syslog RFC series. +
        • +
      • $ModDir
        • +
      • $ModLoad
        • +
      • $RepeatedMsgReduction
        • +
      • $ResetConfigVariables
        • +
      • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better +performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The +default may change as uniprocessor systems become less common.
        • +
      • $WorkDirectory <name> (directory for spool and other work files)
        • +
      • $UDPServerAddress <IP> (imudp) -- local IP +address (or name) the UDP listens should bind to
        • +
      • $UDPServerRun <port> (imudp) -- former +-r<port> option, default 514, start UDP server on this +port, "*" means all addresses
        • +
      • $UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance +optimization. Getting the system time is very costly. With this setting, imudp can +be instructed to obtain the precise time only once every n-times. This logic is +only activated if messages come in at a very fast rate, so doing less frequent +time calls should usually be acceptable. The default value is two, because we have +seen that even without optimization the kernel often returns twice the identical time. +You can set this value as high as you like, but do so at your own risk. The higher +the value, the less precise the timestamp. +
      • $UMASK
        • +
      +

      Where <size_nbr> is specified above, +modifiers can be used after the number part. For example, 1k means +1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa). +Lower case letters refer to the traditional binary defintion (e.g. 1m +equals 1,048,576) whereas upper case letters refer to their new +1000-based definition (e.g 1M equals 1,000,000).

      +

      Numbers may include '.' and ',' for readability. So you can +for example specify either "1000" or "1,000" with the same result. +Please note that rsyslogd simply ignores the punctuation. Form it's +point of view, "1,,0.0.,.,0" also has the value 1000.

      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html new file mode 100644 index 00000000..890a55c8 --- /dev/null +++ b/doc/rsyslog_conf_modules.html @@ -0,0 +1,53 @@ + +Modules - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Modules

      +

      Rsyslog has a modular design. Consequently, there is a growing +number of modules. Here is the entry point to their documentation and +what they do (list is currently not complete)

      +
        +
      • omsnmp - SNMP +trap output module
        • +
      • omrelp - RELP +output module
        • +
      • omgssapi - output module for GSS-enabled syslog
        • +
      • ommysql - output module for MySQL
        • +
      • ompgsql - output module for PostgreSQL
        • +
      • omlibdbi - +generic database output module (Firebird/Interbase, MS SQL, Sybase, +SQLLite, Ingres, Oracle, mSQL)
        • +
      • ommail - +permits rsyslog to alert folks by mail if something important happens
        • +
      • imfile +-  input module for text files
        • +
      • imrelp - RELP +input module
        • +
      • imudp - udp syslog message input
        • +
      • imtcp - input +plugin for plain tcp syslog
        • +
      • imgssapi - +input plugin for plain tcp and GSS-enabled syslog
        • +
      • immark - support for mark messages
        • +
      • imklog - kernel logging
        • +
      • imuxsock - +unix sockets, including the system log socket
        • +
      • im3195 - +accepts syslog messages via RFC 3195
        • +
      +

      Please note that each module provides configuration +directives, which are NOT necessarily being listed below. Also +remember, that a modules configuration directive (and functionality) is +only available if it has been loaded (using $ModLoad).

      +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_output.html b/doc/rsyslog_conf_output.html new file mode 100644 index 00000000..c52aaa5e --- /dev/null +++ b/doc/rsyslog_conf_output.html @@ -0,0 +1,81 @@ + +Output Channels - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Output Channels

      +

      Output Channels are a new concept first introduced in rsyslog +0.9.0. As of this writing, it is most likely that they will +be replaced by something different in the future. So if you +use them, be prepared to change you configuration file syntax when you +upgrade to a later release.
      +
      +The idea behind output channel definitions is that it shall provide an +umbrella for any type of output that the user might want. In essence,
      +this is the "file" part of selector lines (and this is why we are not +sure output channel syntax will stay after the next review). There is a
      +difference, though: selector channels both have filter conditions +(currently facility and severity) as well as the output destination. +they can only be used to write to files - not pipes, ttys or whatever +Output channels define the output definition, only. As of this build, +else. If we stick with output channels, this will change over time.

      +

      In concept, an output channel includes everything needed to +know about an output actions. In practice, the current implementation +only carries
      +a filename, a maximum file size and a command to be issued when this +file size is reached. More things might be present in future version, +which might also change the syntax of the directive.

      +

      Output channels are defined via an $outchannel directive. It's +syntax is as follows:
      +
      +$outchannel name,file-name,max-size,action-on-max-size
      +
      +name is the name of the output channel (not the file), file-name is the +file name to be written to, max-size the maximum allowed size and +action-on-max-size a command to be issued when the max size is reached. +This command always has exactly one parameter. The binary is that part +of action-on-max-size before the first space, its parameter is +everything behind that space.
      +
      +Please note that max-size is queried BEFORE writing the log message to +the file. So be sure to set this limit reasonably low so that any +message might fit. For the current release, setting it 1k lower than +you expected is helpful. The max-size must always be specified in bytes +- there are no special symbols (like 1k, 1m,...) at this point of +development.
      +
      +Keep in mind that $outchannel just defines a channel with "name". It +does not activate it. To do so, you must use a selector line (see +below). That selector line includes the channel name plus an $ sign in +front of it. A sample might be:
      +
      +*.* $mychannel
      +
      +In its current form, output channels primarily provide the ability to +size-limit an output file. To do so, specify a maximum size. When this +size is reached, rsyslogd will execute the action-on-max-size command +and then reopen the file and retry. The command should be something +like a log rotation +script or a similar thing.

      +

      If there is no action-on-max-size command or the command did +not resolve the situation, the file is closed and never reopened by +rsyslogd (except, of course, by huping it). This logic was integrated +when we first experienced severe issues with files larger 2gb, which +could lead to rsyslogd dumping core. In such cases, it is more +appropriate to stop writing to a single file. Meanwhile, rsyslogd has +been fixed to support files larger 2gb, but obviously only on file +systems and operating system versions that do so. So it can still make +sense to enforce a 2gb file size limit.

      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + + diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html new file mode 100644 index 00000000..90b5fafe --- /dev/null +++ b/doc/rsyslog_conf_templates.html @@ -0,0 +1,150 @@ + +Templates - rsyslog.conf + +

      This is a part of the rsyslog.conf - documentation.

      +back +

      Templates

      +

      Templates are a key feature of rsyslog. They allow to specify +any +format a user might want. They are also used for dynamic file name +generation. Every output in rsyslog uses templates - this holds true +for files, user messages and so on. The database writer expects its +template to be a proper SQL statement - so this is highly customizable +too. You might ask how does all of this work when no templates at all +are specified. Good question ;) The answer is simple, though. Templates +compatible with the stock syslogd formats are hardcoded into rsyslogd. +So if no template is specified, we use one of these hardcoded +templates. Search for "template_" in syslogd.c and you will find the +hardcoded ones.

      +

      A template consists of a template directive, a name, the +actual template text and optional options. A sample is:

      +
      $template MyTemplateName,"\7Text +%property% some more text\n",<options>
      +

      The "$template" is the template directive. It tells rsyslog +that this line contains a template. "MyTemplateName" is the template +name. All +other config lines refer to this name. The text within quotes is the +actual template text. The backslash is an escape character, much as it +is in C. It does all these "cool" things. For example, \7 rings the +bell (this is an ASCII value), \n is a new line. C programmers and perl +coders have the advantage of knowing this, but the set in rsyslog is a +bit restricted currently. +

      +

      All text in the template is used literally, except for things +within percent signs. These are properties and allow you access to the +contents of the syslog message. Properties are accessed via the +property replacer (nice name, huh) and it can do cool things, too. For +example, it can pick a substring or do date-specific formatting. More +on this is below, on some lines of the property replacer.
      +
      +The <options> part is optional. It carries options +influencing the template as whole. See details below. Be sure NOT to +mistake template options with property options - the later ones are +processed by the property replacer and apply to a SINGLE property, only +(and not the whole template).
      +
      +Template options are case-insensitive. Currently defined are:

      +

      sql - format the string suitable for a SQL +statement in MySQL format. This will replace single quotes ("'") and +the backslash character by their backslash-escaped counterpart ("\'" +and "\\") inside each field. Please note that in MySQL configuration, +the NO_BACKSLASH_ESCAPES +mode must be turned off for this format to work (this is the default).

      +

      stdsql - format the string suitable for a +SQL statement that is to be sent to a standards-compliant sql server. +This will replace single quotes ("'") by two single quotes ("''") +inside each field. You must use stdsql together with MySQL if in MySQL +configuration the +NO_BACKSLASH_ESCAPES is +turned on.

      +

      Either the sql or stdsql  +option must be specified when a template is used +for writing to a database, otherwise injection might occur. Please note +that due to the unfortunate fact that several vendors have violated the +sql standard and introduced their own escape methods, it is impossible +to have a single option doing all the work.  So you yourself +must make sure you are using the right format. If you choose +the wrong one, you are still vulnerable to sql injection.
      +
      +Please note that the database writer *checks* that the sql option is +present in the template. If it is not present, the write database +action is disabled. This is to guard you against accidental forgetting +it and then becoming vulnerable to SQL injection. The sql option can +also be useful with files - especially if you want to import them into +a database on another machine for performance reasons. However, do NOT +use it if you do not have a real need for it - among others, it takes +some toll on the processing time. Not much, but on a really busy system +you might notice it ;)

      +

      The default template for the write to database action has the +sql option set. As we currently support only MySQL and the sql option +matches the default MySQL configuration, this is a good choice. +However, if you have turned on +NO_BACKSLASH_ESCAPES in +your MySQL config, you need to supply a template with the stdsql +option. Otherwise you will become vulnerable to SQL injection.
      +
      +To escape:
      +% = \%
      +\ = \\ --> '\' is used to escape (as in C)
      +$template TraditionalFormat,%timegenerated% %HOSTNAME% +%syslogtag%%msg%\n"
      +
      +Properties can be accessed by the property +replacer (see there for details).

      +

      Please note that templates can also by +used to generate selector lines with dynamic file names. For +example, if you would like to split syslog messages from different +hosts to different files (one per host), you can define the following +template:

      +
      $template +DynFile,"/var/log/system-%HOSTNAME%.log"
      +

      This template can then be used when defining an output +selector line. It will result in something like +"/var/log/system-localhost.log"

      +

      Template +names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT +use them if, otherwise you may receive a conflict in the future (and +quite unpredictable behaviour). There is a small set of pre-defined +templates that you can use without the need to define it:

      +
        +
      • RSYSLOG_TraditionalFileFormat +- the "old style" default log file format with low-precision timestamps
        • +
      • RSYSLOG_FileFormat +- a modern-style logfile format similar to TraditionalFileFormat, buth +with high-precision timestamps and timezone information
        • +
      • RSYSLOG_TraditionalForwardFormat +- the traditional forwarding format with low-precision timestamps. Most +useful if you send messages to other syslogd's or rsyslogd +below +version 3.12.5.
        • +
      • RSYSLOG_ForwardFormat +- a new high-precision forwarding format very similar to the +traditional one, but with high-precision timestamps and timezone +information. Recommended to be used when sending messages to rsyslog +3.12.5 or above.
        • +
      • RSYSLOG_SyslogProtocol23Format +- the format specified in IETF's internet-draft +ietf-syslog-protocol-23, which is assumed to be come the new syslog +standard RFC. This format includes several improvements. The rsyslog +message parser understands this format, so you can use it together with +all relatively recent versions of rsyslog. Other syslogd's may get +hopelessly confused if receiving that format, so check before you use +it. Note that the format is unlikely to change when the final RFC comes +out, but this may happen.
        • +
      • RSYSLOG_DebugFormat +- a special format used for troubleshooting property problems. This format +is meant to be written to a log file. Do not use for production or remote +forwarding.
        • +
      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + -- cgit From 249b27952a9faea95662eb230f4c86a0db874fe5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 11 Nov 2008 11:38:37 +0100 Subject: improved doc on property replacer regular expressions --- doc/Makefile.am | 1 + doc/property_replacer.html | 10 +++------- doc/rsyslog_conf_nomatch.html | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 doc/rsyslog_conf_nomatch.html diff --git a/doc/Makefile.am b/doc/Makefile.am index fef1e44c..5c2f5313 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -105,6 +105,7 @@ html_files = \ rsyslog_conf_modules.html \ rsyslog_conf_output.html \ rsyslog_conf_templates.html \ + rsyslog_conf_nomatch.html \ src/classes.dia EXTRA_DIST = $(html_files) diff --git a/doc/property_replacer.html b/doc/property_replacer.html index 34e2116c..9ea41aed 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -229,7 +229,7 @@ sequence with a regular expression is: "%msg:R:.*Sev:. \(.*\) \[.*--end%"

      It is possible to specify some parametes after the "R". These are comma-separated. They are: -

      R,<regexp-type>,<submatch>,<nomatch>,<match-number> +

      R,<regexp-type>,<submatch>,<nomatch>,<match-number>

      regexp-type is either "BRE" for Posix basic regular expressions or "ERE" for extended ones. The string must be given in upper case. The default is "BRE" to be consistent with earlier versions of rsyslog that @@ -241,12 +241,8 @@ that the first match is number 0, the second 1 and so on. Up to 10 matches (up to number 9) are supported. Please note that it would be more natural to have the match-number in front of submatch, but this would break backward-compatibility. So the match-number must be specified after "nomatch". -

      nomatch is either "DFLT", "BLANK" or "FIELD" (all upper case!). It tells -what to use if no match is found. With "DFLT", the strig "**NO MATCH**" is -used. This was the only supported value up to rsyslog 3.19.5. With "BLANK" -a blank text is used (""). Finally, "FIELD" uses the full property text -instead of the expression. Some folks have requested that, so it seems -to be useful. +

      nomatch specifies what should +be used in case no match is found.

      The following is a sample of an ERE expression that takes the first submatch from the message string and replaces the expression with the full field if no match is found: diff --git a/doc/rsyslog_conf_nomatch.html b/doc/rsyslog_conf_nomatch.html new file mode 100644 index 00000000..5c4f3f90 --- /dev/null +++ b/doc/rsyslog_conf_nomatch.html @@ -0,0 +1,37 @@ + +nomatch mode - property replacer - rsyslog.conf + +

      nomatch mode - property replacer - rsyslog.con

      +

      This is a part of the rsyslog.conf documentation +of the property replacer.

      +

      The "nomatch-Mode" specifies which string the property replacer +shall return if a regular expression did not find the search string.. Traditionally, +the string "**NO MATCH**" was returned, but many people complained this was almost never useful. +Still, this mode is support as "DFLT" for legacy configurations. +

      Two additional and potentially useful modes exist: in one (BLANK) a blank string +is returned. This is probably useful for inserting values into databases where no +value shall be inserted if the expression could not be found. A use case may be +that you record a traffic log based on firewall rules and the "bytes transmitted" counter +is extracted via a regular expression. If no "bytes transmitted" counter is available +in the current message, it is probably a good idea to return an empty string, which the +database layer can turn into a zero. +

      The other mode is "FIELD", in which the complete field is returned. This may be useful +in cases where absense of a match is considered a failure and the message that triggered +it shall be logged. +

      If in doubt, it is highly suggested to use the +rsyslog online regular expression +checker and generator to see these options in action. With that online tool, +you can craft regular expressions based on samples and try out the different modes. + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + + -- cgit From 704a1145d64532df36624a3c9850b0c86f38f76f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 11 Nov 2008 12:22:06 +0100 Subject: doc: added new nomatch mode Note: the actual nomatch mode is not yet available in this code, this needs to be merged from v3-stable first. This happens soon, but I wanted to make sure the doc is right. --- doc/rsyslog_conf_nomatch.html | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/rsyslog_conf_nomatch.html b/doc/rsyslog_conf_nomatch.html index 5c4f3f90..5f25f3e4 100644 --- a/doc/rsyslog_conf_nomatch.html +++ b/doc/rsyslog_conf_nomatch.html @@ -8,9 +8,11 @@ of the property replacer.

      shall return if a regular expression did not find the search string.      . Traditionally, the string "**NO MATCH**" was returned, but many people complained this was almost never useful. Still, this mode is support as "DFLT" for legacy configurations. -

      Two additional and potentially useful modes exist: in one (BLANK) a blank string +

      Three additional and potentially useful modes exist: in one (BLANK) a blank string is returned. This is probably useful for inserting values into databases where no -value shall be inserted if the expression could not be found. A use case may be +value shall be inserted if the expression could not be found. +

      A similar mode is "ZERO" where the string "0" is returned. This is suitable +for numerical values. A use case may be that you record a traffic log based on firewall rules and the "bytes transmitted" counter is extracted via a regular expression. If no "bytes transmitted" counter is available in the current message, it is probably a good idea to return an empty string, which the @@ -23,6 +25,15 @@ it shall be logged. checker and generator to see these options in action. With that online tool, you can craft regular expressions based on samples and try out the different modes. +

      Summary of nomatch Modes

      + + + + + + + +
      ModeReturned
      DFLT"**NO MATCH**"
      BLANK"" (empty string)
      ZERO"0"
      FIELDfull content of original field
       Interactive Tool

      [manual index] [rsyslog.conf] [rsyslog site]

      -- cgit From afd425232f3198e7bb6f3dbcf0aa3cb4c24d5f7e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 18 Nov 2008 13:46:38 +0100 Subject: enhanced legacy syslog parser to detect year if part of the timestamp The format is based on what Cisco devices seem to emit. --- ChangeLog | 5 +++++ runtime/datetime.c | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/ChangeLog b/ChangeLog index f758b7ce..70ad7e1f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ version before switching to this one. - added configuration directive "HUPisRestart" which enables to configure HUP to be either a full restart or "just" a leightweight way to close open files. +- enhanced legacy syslog parser to detect year if part of the timestamp + the format is based on what Cisco devices seem to emit. - added a setting "$OptimizeForUniprocessor" to enable users to turn off pthread_yield calls which are counter-productive on multiprocessor machines (but have been shown to be useful on uniprocessors) @@ -29,6 +31,8 @@ version before switching to this one. - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period - restructured rsyslog.conf documentation +- bugfix: memory leak in ompgsql + Thanks to Ken for providing the patch --------------------------------------------------------------------------- Version 3.21.7 [BETA] (rgerhards), 2008-11-11 - this is the new beta branch, based on the former 3.21.6 devel @@ -45,6 +49,7 @@ Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - added capability to support multiple module search pathes. Thank to Marius Tomaschewski for providing the patch. - bugfix: im3195 did no longer compile +- improved "make distcheck" by ensuring everything relevant is recompiled --------------------------------------------------------------------------- Version 3.21.5 [DEVEL] (rgerhards), 2008-09-30 - performance optimization: unnecessary time() calls during message diff --git a/runtime/datetime.c b/runtime/datetime.c index aa1956d7..676f76d5 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -312,6 +312,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) /* variables to temporarily hold time information while we parse */ int month; int day; + int year = 0; /* 0 means no year provided */ int hour; /* 24 hour clock */ int minute; int second; @@ -472,7 +473,23 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) if(*pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); + + /* time part */ hour = srSLMGParseInt32(&pszTS); + if(hour > 1970 && hour < 2100) { + /* if so, we assume this actually is a year. This is a format found + * e.g. in Cisco devices. + * (if you read this 2100+ trying to fix a bug, congratulate myself + * to how long the code survived - me no longer ;)) -- rgerhards, 2008-11-18 + */ + year = hour; + + /* re-query the hour, this time it must be valid */ + if(*pszTS++ != ' ') + ABORT_FINALIZE(RS_RET_INVLD_TIME); + hour = srSLMGParseInt32(&pszTS); + } + if(hour < 0 || hour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); @@ -502,6 +519,8 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) *ppszTS = pszTS; /* provide updated parse position back to caller */ pTime->timeType = 1; pTime->month = month; + if(year > 0) + pTime->year = year; /* persist year if detected */ pTime->day = day; pTime->hour = hour; pTime->minute = minute; -- cgit From 57c9a3accee3a3e9b46d984c76c9aae7e2ec9c27 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 19 Nov 2008 13:20:31 +0100 Subject: exprimental implementaiton of $PrivDropToUser directive ... which permits to drop root privileges. This is not a completely secure way of dropping permissions, e.g. the group permissions need to be dropped, too. Also, there are several vulnerability Windows (see code comments). Finally, at least the imklog module on linux does not work when privileges are dropped. This code may still be a valuable addition, and so I have created an experimental branch so that people can check it out. --- tools/syslogd.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index 7145779d..51ce1830 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -272,6 +272,7 @@ static int bHaveMainQueue = 0;/* set to 1 if the main queue - in queueing mode - * If the main queue is either not yet ready or not running in * queueing mode (mode DIRECT!), then this is set to 0. */ +static int uidDropPriv = 0; /* user-id to which priveleges should be dropped to (AFTER init()!) */ extern int errno; @@ -2063,6 +2064,30 @@ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) } +/* drop to specified user + * if something goes wrong, the function never returns + * Note that such an abort can cause damage to on-disk structures, so we should + * re-design the "interface" in the long term. -- rgerhards, 2008-11-19 + */ +static void doDropPrivUid(int iUid) +{ + int res; + uchar szBuf[1024]; + +dbgprintf("userid before drop: %d\n", getuid()); + res = setuid(iUid); +dbgprintf("userid after drop: %d\n", getuid()); + if(res) { + /* if we can not set the userid, this is fatal, so let's unconditionally abort */ + perror("could not set requested userid"); + exit(1); + } + DBGPRINTF("setuid(%d): %d\n", iUid, res); + snprintf((char*)szBuf, sizeof(szBuf)/sizeof(uchar), "rsyslogd's userid changed to %d", iUid); + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); +} + + /* helper to freeSelectors(), used with llExecFunc() to flush * pending output. -- rgerhards, 2007-08-02 * We do not need to lock the action object here as the processing @@ -2807,6 +2832,8 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"errormessagestostderr", 0, eCmdHdlrBinary, NULL, &bErrMsgToStderr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, setMaxMsgSize, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptouser", 0, eCmdHdlrUID, NULL, &uidDropPriv, NULL)); +// CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, doDropPrivGroup, NULL, NULL)); /* now add other modules handlers (we should work on that to be able to do it in ClassInit(), but so far * that is not possible). -- rgerhards, 2008-01-28 @@ -2900,8 +2927,20 @@ static rsRetVal mainThread() } /* Send a signal to the parent so it can terminate. */ - if (myPid != ppid) - kill (ppid, SIGTERM); + if(myPid != ppid) + kill(ppid, SIGTERM); + + + /* If instructed to do so, we now drop privileges. Note that this is not 100% secure, + * because inputs and outputs are already running at this time. However, we can implement + * dropping of privileges rather quickly and it will work in many cases. While it is not + * the ultimate solution, the current one is still much better than not being able to + * drop privileges at all. Doing it correctly, requires a change in architecture, which + * we should do over time. TODO -- rgerhards, 2008-11-19 + */ + if(uidDropPriv != 0) { + doDropPrivUid(uidDropPriv); + } /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might @@ -3520,11 +3559,14 @@ int realMain(int argc, char **argv) /* process compatibility mode settings */ - if(iCompatibilityMode < 3) { + if(iCompatibilityMode < 4) { errmsg.LogError(0, NO_ERRCODE, "WARNING: rsyslogd is running in compatibility mode. Automatically " "generated config directives may interfer with your rsyslog.conf settings. " - "We suggest upgrading your config and adding -c3 as the first " + "We suggest upgrading your config and adding -c4 as the first " "rsyslogd option."); + } + + if(iCompatibilityMode < 3) { if(MarkInterval > 0) { legacyOptsEnq((uchar *) "ModLoad immark"); snprintf((char *) legacyConfLine, sizeof(legacyConfLine), "MarkMessagePeriod %d", MarkInterval); -- cgit From 48480e833488c8bc571c62ad84331038edddf3c3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 21 Nov 2008 15:38:34 +0100 Subject: improved doc a big --- doc/manual.html | 3 +++ doc/queues.html | 79 +++++++++++++++++++++++++-------------------------------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/doc/manual.html b/doc/manual.html index e208755d..1762fcca 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,6 +16,9 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

      +

      Please visit the rsyslog sponsor's page +to honor the project sponsor or become one yourself! We are very grateful for any help towards the +project goals.

      This documentation is for version 4.1.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status. diff --git a/doc/queues.html b/doc/queues.html index 727bc26a..41c5865f 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -17,6 +17,36 @@ rule processor, which then evaluates which actions are to be carried out. In front of each action, there is also a queue, which potentially de-couples the filter processing from the actual action (e.g. writing to file, database or forwarding to another host).

      +

      Where are Queues Used?

      +

       Currently, queues are used for the main message queue and for the +actions.

      +

      There is a single main message queue inside rsyslog. Each input module +delivers messages to it. The main message queue worker filters messages based on +rules specified in rsyslog.conf and dispatches them to the individual action +queues. Once a message is in an action queue, it is deleted from the main +message queue.

      +

      There are multiple action queues, one for each configured action. By default, +these queues operate in direct (non-queueing) mode. Action queues are fully +configurable and thus can be changed to whatever is best for the given use case.

      +

      Future versions of rsyslog will most probably utilize queues at other places, +too.

      +

      Wherever "<object>"  is used in the config file +statements, substitute "<object>" with either "MainMsg" or "Action". The +former will set main message queue +parameters, the later parameters for the next action that will be +created. Action queue parameters can not be modified once the action has been +specified. For example, to tell the main message queue to save its content on +shutdown, use $MainMsgQueueSaveOnShutdown on".

      +

      If the same parameter is specified multiple times before a queue is created, +the last one specified takes precedence. The main message queue is created after +parsing the config file and all of its potential includes. An action queue is +created each time an action selector is specified. Action queue parameters are +reset to default after an action queue has been created (to provide a clean +environment for the next action).

      +

      Not all queues necessarily support the full set of queue configuration +parameters, because not all are applicable. For example, in current output +module design, actions do not support multi-threading. Consequently, the number +of worker threads is fixed to one for action queues and can not be changed.

      Queue Modes

      Rsyslog supports different queue modes, some with submodes. Each of them has specific advantages and disadvantages. Selecting the right queue mode is quite @@ -114,8 +144,7 @@ only memory if in use. A FixedArray queue may have a too large static memory footprint in such cases.

      In general, it is advised to use LinkedList mode if in doubt. The processing overhead compared to FixedArray is low and may be - -outweigh by the reduction in memory use. Paging in most-often-unused +outweigh by the reduction in memory use. Paging in most-often-unused pointer array pages can be much slower than dynamically allocating them.

      To create an in-memory queue, use the "$<object>QueueType LinkedList" or  "$<object>QueueType FixedArray" config directive.

      @@ -260,19 +289,15 @@ unavoidable and you prefer to discard less important messages first.

      disk space, it is finally full. If so, rsyslogd throttles the data element submitter. If that, for example, is a reliable input (TCP, local log socket), that will slow down the message originator which is a good - -resolution for this scenario.

      -

      During - -throtteling, a disk-assisted queue continues to write to disk and +resolution for this scenario.

      +

      During throtteling, a disk-assisted queue continues to write to disk and messages are also discarded based on severity as well as regular dequeuing and processing continues. So chances are good the situation will be resolved by simply throttling. Note, though, that throtteling is highly undesirable for unreliable sources, like UDP message reception. So it is not a good thing to run into throtteling mode at all.

      We can not hold processing - -infinitely, not even when throtteling. For example, throtteling the local +infinitely, not even when throtteling. For example, throtteling the local log socket too long would cause the system at whole come to a standstill. To prevent this, rsyslogd times out after a configured period ("$<object>QueueTimeoutEnqueue", specified in milliseconds) if no space becomes available. As a last resort, it @@ -303,8 +328,7 @@ There are two configuration directives, both should be used together or results are unpredictable:" $<object>QueueDequeueTimeBegin <hour>" and "$<object>QueueDequeueTimeEnd <hour>". The hour parameter must be specified in 24-hour format (so 10pm is 22). A use case for this parameter can be found in the rsyslog wiki.

      Terminating Queues

      Terminating a process sounds easy, but can be complex. - -Terminating a running queue is in fact the most complex operation a queue +Terminating a running queue is in fact the most complex operation a queue object can perform. You don't see that from a user's point of view, but its quite hard work for the developer to do everything in the right order.

      The complexity arises when the queue has still data enqueued when it @@ -325,39 +349,6 @@ it terminates. This includes data elements there were begun being processed by workers that needed to be cancelled due to too-long processing. For a large queue, this operation may be lengthy. No timeout applies to a required shutdown save.

      -

      Where are Queues Used?

      -

       Currently, queues are used for the main message queue and for the -actions.

      -

      There is a single main message queue inside rsyslog. Each input module -delivers messages to it. The main message queue worker filters messages based on -rules specified in rsyslog.conf and dispatches them to the individual action -queues. Once a message is in an action queue, it is deleted from the main -message queue.

      -

      There are multiple action queues, one for each configured action. By default, -these queues operate in direct (non-queueing) mode. Action queues are fully -configurable and thus can be changed to whatever is best for the given use case.

      -

      Future versions of rsyslog will most probably utilize queues at other places, -too.

      -

      - -Wherever "<object>"  was used above in the config file -statements, substitute "<object>" with either "MainMsg" or "Action". The -former will set main message queue - -parameters, the later parameters for the next action that will be -created. Action queue parameters can not be modified once the action has been -specified. For example, to tell the main message queue to save its content on -shutdown, use $MainMsgQueueSaveOnShutdown on".

      -

      If the same parameter is specified multiple times before a queue is created, -the last one specified takes precedence. The main message queue is created after -parsing the config file and all of its potential includes. An action queue is -created each time an action selector is specified. Action queue parameters are -reset to default after an action queue has been created (to provide a clean -environment for the next action).

      -

      Not all queues necessarily support the full set of queue configuration -parameters, because not all are applicable. For example, in current output -module design, actions do not support multi-threading. Consequently, the number -of worker threads is fixed to one for action queues and can not be changed.

      [manual index] [rsyslog.conf] [rsyslog site]

      -- cgit From 588f4f98d8b484ea6b2cba3449db9c9b035b562f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 21 Nov 2008 15:41:34 +0100 Subject: doc bugfix: property-based filter had inconsistent description Thank to Peter Matulis for pointing this out. Actually, all compare operations described exist. --- doc/rsyslog_conf_filter.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html index 55244c15..cf21ff95 100644 --- a/doc/rsyslog_conf_filter.html +++ b/doc/rsyslog_conf_filter.html @@ -97,9 +97,7 @@ filter on any property, like HOSTNAME, syslogtag and msg. A list of all currently-supported properties can be found in the property replacer documentation (but keep in mind that only the properties, not the replacer is supported). With this filter, each properties can be checked against a -specified value, using a specified compare operation. Currently, there -is only a single compare operation (contains) available, but additional -operations will be added in the future.

      +specified value, using a specified compare operation.

      A property-based filter must start with a colon in column 0. This tells rsyslogd that it is the new filter type. The colon must be followed by the property name, a comma, the name of the compare -- cgit From 5f44f8592761161921ceefffeedadbec2c308453 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 11:21:29 +0100 Subject: improved doc a bit --- doc/features.html | 7 +++++++ doc/manual.html | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/features.html b/doc/features.html index 89796a41..7a31a03b 100644 --- a/doc/features.html +++ b/doc/features.html @@ -116,6 +116,13 @@ submit feature requests there (or via our forums). If we like them but they look quite long-lived (aka "not soon to be implemented"), they will possibly be migrated to this list here and at some time moved back to the bugzilla tracker.

      +

      Note that we also maintain a +list of features that are looking for sponsors. +If you are interested in any of these features, or any other feature, you may consider sponsoring +the implementation. This is also a great way to show your commitment to the open source +community. Plus, it can be financially attractive: just think about how much less it may +be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit +of being recognised as a sponsor may even drive new customers to your business!

      • port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge
        • diff --git a/doc/manual.html b/doc/manual.html index 1762fcca..dee076b8 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,8 +16,8 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

        -

        Please visit the rsyslog sponsor's page -to honor the project sponsor or become one yourself! We are very grateful for any help towards the +

        Please visit the rsyslog sponsor's page +to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

        This documentation is for version 4.1.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current -- cgit From dc478db1ca80ef222f83985b539dfec1c66063e2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 14:17:36 +0100 Subject: added ability to drop privileges Added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, $PrivDropToUserID config directives to enable dropping privileges. This is an effort to provide a security enhancement. For the limits of this approach, see http://wiki.rsyslog.com/index.php/Security --- ChangeLog | 6 +++++ configure.ac | 2 +- doc/Makefile.am | 1 + doc/droppriv.html | 60 ++++++++++++++++++++++++++++++++++++++++++++ doc/manual.html | 2 +- doc/rsyslog_conf_global.html | 5 ++++ tools/syslogd.c | 43 ++++++++++++++++++++++++++++--- 7 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 doc/droppriv.html diff --git a/ChangeLog b/ChangeLog index 70ad7e1f..fe8df1fc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ --------------------------------------------------------------------------- +Version 4.1.1 [DEVEL] (rgerhards), 2008-11-?? +- added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, + $PrivDropToUserID config directives to enable dropping privileges. + This is an effort to provide a security enhancement. For the limits of this + approach, see http://wiki.rsyslog.com/index.php/Security +--------------------------------------------------------------------------- Version 4.1.0 [DEVEL] (rgerhards), 2008-11-18 ********************************* WARNING ********************************* diff --git a/configure.ac b/configure.ac index 981ef193..8eecf2ab 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/Makefile.am b/doc/Makefile.am index 5c2f5313..b58f813b 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -5,6 +5,7 @@ html_files = \ features.html \ generic_design.html \ expression.html \ + droppriv.html \ history.html \ how2help.html \ install.html \ diff --git a/doc/droppriv.html b/doc/droppriv.html new file mode 100644 index 00000000..7293e872 --- /dev/null +++ b/doc/droppriv.html @@ -0,0 +1,60 @@ + +dropping privileges in rsyslog + + +

        Dropping privileges in rsyslog

        +

        Available since:    4.1.1

        +

        Description:

        +

        +Rsyslogd provides the ability to drop privileges by +impersonating as another user and/or group after startup. + +

        Please note that due to POSIX standards, rsyslogd always needs to start +up as root if there is a listener who must bind to a network port below 1024. +For example, the UDP listener usually needs to listen to 514 and as such +rsyslogd needs to start up as root. + +

        If you do not need this functionality, you can start rsyslog directly as an ordinary +user. That is probably the safest way of operations. However, if a startup as +root is required, you can use the $PrivDropToGroup and $PrivDropToUser config +directives to specify a group and/or user that rsyslogd should drop to after initialization. +Once this happend, the daemon runs without high privileges (depending, of +course, on the permissions of the user account you specified). +

        There is some additional information available in the +rsyslog wiki. +

        Configuration Directives:

        +
          +
        • $PrivDropToUser
          +Name of the user rsyslog should run under after startup. Please note that +this user is looked up in the system tables. If the lookup fails, privileges are +NOT dropped. Thus it is advisable to use the less convenient $PrivDropToUserID directive. +If the user id can be looked up, but can not be set, rsyslog aborts. +
          +
          • +
        • $PrivDropToUserID
          +Much the same as $PrivDropToUser, except that a numerical user id instead of a name +is specified.Thus, privilege drop will always happen. +rsyslogd aborts. +
        • $PrivDropToGroup
          +Name of the group rsyslog should run under after startup. Please note that +this user is looked up in the system tables. If the lookup fails, privileges are +NOT dropped. Thus it is advisable to use the less convenient $PrivDropToGroupID directive. +Note that all supplementary groups are removed from the process if $PrivDropToGroup is +specified. +If the group id can be looked up, but can not be set, rsyslog aborts. +
          +
          • +
        • $PrivDropToGroupID
          +Much the same as $PrivDropToGroup, except that a numerical group id instead of a name +is specified. Thus, privilege drop will always happen. +
        +

        [rsyslog.conf overview] +[manual index] [rsyslog site]

        +

        This documentation is part of the rsyslog +project.
        +Copyright © 2008 by Rainer +Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

        + + diff --git a/doc/manual.html b/doc/manual.html index e208755d..8210165f 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,7 +16,7 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

        -

        This documentation is for version 4.1.0 (devel branch) of rsyslog. +

        This documentation is for version 4.1.1 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

        If you like rsyslog, you might diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index bc618dd0..d02245e3 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -200,6 +200,11 @@ time calls should usually be acceptable. The default value is two, because we ha seen that even without optimization the kernel often returns twice the identical time. You can set this value as high as you like, but do so at your own risk. The higher the value, the less precise the timestamp. +

      • $PrivDropToGroup
        • +
      • $PrivDropToGroupID
        • +
      • $PrivDropToUser
        • +
      • $PrivDropToUserID
        • +
    • $UMASK

    Where <size_nbr> is specified above, diff --git a/tools/syslogd.c b/tools/syslogd.c index 51ce1830..d132d139 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -82,6 +82,7 @@ #include #include #include +#include #if HAVE_SYS_TIMESPEC_H # include @@ -273,6 +274,7 @@ static int bHaveMainQueue = 0;/* set to 1 if the main queue - in queueing mode - * queueing mode (mode DIRECT!), then this is set to 0. */ static int uidDropPriv = 0; /* user-id to which priveleges should be dropped to (AFTER init()!) */ +static int gidDropPriv = 0; /* group-id to which priveleges should be dropped to (AFTER init()!) */ extern int errno; @@ -2064,6 +2066,34 @@ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) } +/* drop to specified group + * if something goes wrong, the function never returns + * Note that such an abort can cause damage to on-disk structures, so we should + * re-design the "interface" in the long term. -- rgerhards, 2008-11-26 + */ +static void doDropPrivGid(int iGid) +{ + int res; + uchar szBuf[1024]; + + res = setgroups(0, NULL); /* remove all supplementary group IDs */ + if(res) { + perror("could not remove supplemental group IDs"); + exit(1); + } + DBGPRINTF("setgroups(0, NULL): %d\n", res); + res = setgid(iGid); + if(res) { + /* if we can not set the userid, this is fatal, so let's unconditionally abort */ + perror("could not set requested group id"); + exit(1); + } + DBGPRINTF("setgid(%d): %d\n", iGid, res); + snprintf((char*)szBuf, sizeof(szBuf)/sizeof(uchar), "rsyslogd's groupid changed to %d", iGid); + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); +} + + /* drop to specified user * if something goes wrong, the function never returns * Note that such an abort can cause damage to on-disk structures, so we should @@ -2074,9 +2104,7 @@ static void doDropPrivUid(int iUid) int res; uchar szBuf[1024]; -dbgprintf("userid before drop: %d\n", getuid()); res = setuid(iUid); -dbgprintf("userid after drop: %d\n", getuid()); if(res) { /* if we can not set the userid, this is fatal, so let's unconditionally abort */ perror("could not set requested userid"); @@ -2833,7 +2861,9 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"errormessagestostderr", 0, eCmdHdlrBinary, NULL, &bErrMsgToStderr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, setMaxMsgSize, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"privdroptouser", 0, eCmdHdlrUID, NULL, &uidDropPriv, NULL)); -// CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, doDropPrivGroup, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptouserid", 0, eCmdHdlrInt, NULL, &uidDropPriv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroupid", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); /* now add other modules handlers (we should work on that to be able to do it in ClassInit(), but so far * that is not possible). -- rgerhards, 2008-01-28 @@ -2938,10 +2968,17 @@ static rsRetVal mainThread() * drop privileges at all. Doing it correctly, requires a change in architecture, which * we should do over time. TODO -- rgerhards, 2008-11-19 */ + if(gidDropPriv != 0) { + doDropPrivGid(gidDropPriv); + glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ + } + if(uidDropPriv != 0) { doDropPrivUid(uidDropPriv); + glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ } + /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might * be called again. If that happens, we must shut down the worker thread, -- cgit From d44ae01df3cd41a25523e82f3acc76f048d20aa7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 16:55:05 +0100 Subject: fixing issue with test suite which was not yet adapted to v4 --- ChangeLog | 2 +- tests/cfg.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index 66af15d1..22c6006d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.1.1 [DEVEL] (rgerhards), 2008-11-?? +Version 4.1.1 [DEVEL] (rgerhards), 2008-11-26 - added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, $PrivDropToUserID config directives to enable dropping privileges. This is an effort to provide a security enhancement. For the limits of this diff --git a/tests/cfg.sh b/tests/cfg.sh index 99729823..fb22fbf3 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -36,7 +36,7 @@ echo "local directory" # # check empty config file # -../tools/rsyslogd -c3 -N1 -f/dev/null 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/DevNull.cfgtest if [ ! $? -eq 0 ]; then echo "DevNull.cfgtest failed" @@ -51,7 +51,7 @@ fi; # # check missing config file # -../tools/rsyslogd -c3 -N1 -f/This/does/not/exist 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/NoExistFile.cfgtest if [ ! $? -eq 0 ]; then echo "NoExistFile.cfgtest failed" @@ -74,7 +74,7 @@ exit 0 # # check config with invalid directive # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg1.cfgtest if [ ! $? -eq 0 ]; then echo "cfg1.cfgtest failed" @@ -91,7 +91,7 @@ fi; # the one with the invalid config directive, so that we may see # an effect of the included config ;) # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg2.cfgtest if [ ! $? -eq 0 ]; then echo "cfg2.cfgtest failed" @@ -106,7 +106,7 @@ fi; # # check included config file, where included file does not exist # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg3.cfgtest if [ ! $? -eq 0 ]; then echo "cfg3.cfgtest failed" @@ -121,7 +121,7 @@ fi; # # check a reasonable complex, but correct, log file # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg4.cfgtest if [ ! $? -eq 0 ]; then echo "cfg4.cfgtest failed" -- cgit From 1f1e3254924aff524ed209042e70d3af333b092d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 17:18:22 +0100 Subject: updated project status --- doc/status.html | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/status.html b/doc/status.html index 5c8409ec..b814d0bc 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,19 +2,19 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2008-10-22.

    +

    This page reflects the status as of 2008-11-26.

    Current Releases

    -

    development: 3.21.6 [2008-10-22] - -change log - -download +

    development: 4.1.1 [2008-11-26] - +change log - +download -
    beta: 3.19.12 [2008-10-16] - -change log - -download

    +
    beta: 3.21.7 [2008-11-11] - +change log - +download

    -

    v3 stable: 3.18.5 [2008-10-09] - change log - -download +

    v3 stable: 3.20.0 [2008-11-05] - change log - +download
    v2 stable: 2.0.6 [2008-08-07] - change log - download -- cgit From 6b905b511b685f2ae28ef94d2e0ba14d1a3f4df3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Dec 2008 10:45:11 +0100 Subject: bugfix: code did not compile without zlib --- ChangeLog | 3 +++ runtime/parser.c | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 22c6006d..3ac558e3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 4.1.2 [DEVEL] (rgerhards), 2008-11-?? +- bugfix: code did not compile without zlib +--------------------------------------------------------------------------- Version 4.1.1 [DEVEL] (rgerhards), 2008-11-26 - added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, $PrivDropToUserID config directives to enable dropping privileges. diff --git a/runtime/parser.c b/runtime/parser.c index fbdeebeb..15dfd4e0 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -127,12 +127,16 @@ finalize_it: /* in this case, we still need to check if the message is compressed. If so, we must * tell the user we can not accept it. */ - if(len > 0 && *msg == 'z') { + //pszMsg = pMsg->pszRawMsg; + //lenMsg = pMsg->iLenRawMsg; + //if(lenMsg > 0 && *msg == 'z') { + if(pMsg->iLenRawMsg > 0 && *pMsg->pszRawMsg == 'z') { errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " "support enabled. The message will be ignored."); ABORT_FINALIZE(RS_RET_NO_ZIP); } +finalize_it: # endif /* ifdef USE_NETZIP */ RETiRet; -- cgit From c5bfd2b24ca8c490401a0835ec741c05acf0ed3e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Dec 2008 10:46:27 +0100 Subject: some cleanup (forgotten...) --- runtime/parser.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/runtime/parser.c b/runtime/parser.c index 15dfd4e0..ec2a28c7 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -127,9 +127,6 @@ finalize_it: /* in this case, we still need to check if the message is compressed. If so, we must * tell the user we can not accept it. */ - //pszMsg = pMsg->pszRawMsg; - //lenMsg = pMsg->iLenRawMsg; - //if(lenMsg > 0 && *msg == 'z') { if(pMsg->iLenRawMsg > 0 && *pMsg->pszRawMsg == 'z') { errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " "support enabled. The message will be ignored."); @@ -295,7 +292,7 @@ rsRetVal parseMsg(msg_t *pMsg) if(msg[0] == '1' && msg[1] == ' ') { dbgprintf("Message has syslog-protocol format.\n"); setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { // TODO: parseRFC... should pull flags from pMsg + if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { msgDestruct(&pMsg); ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! } -- cgit From 3e1220f434533b5e91de51f5de17cc76eaa8af45 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Dec 2008 01:10:06 +0100 Subject: fixed some compiler warnings --- dirty.h | 2 ++ runtime/parser.c | 1 + 2 files changed, 3 insertions(+) diff --git a/dirty.h b/dirty.h index 4b13adcd..db9bc31b 100644 --- a/dirty.h +++ b/dirty.h @@ -30,6 +30,8 @@ rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); +int parseRFCSyslogMsg(msg_t *pMsg, int flags); +int parseLegacySyslogMsg(msg_t *pMsg, int flags); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/runtime/parser.c b/runtime/parser.c index ec2a28c7..b549cd19 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -27,6 +27,7 @@ #include "config.h" #include #include +#include #include #ifdef USE_NETZIP #include -- cgit From 85c85c1496856307a0080150176842e9a480a5a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Dec 2008 02:14:20 +0100 Subject: doc bugfix: typo in v3 compatibility document thanks to Andrej for reporting --- ChangeLog | 2 ++ doc/v3compatibility.html | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 43dcbff8..d60e6f76 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- doc bugfix: typo in v3 compatibility document + thanks to Andrej for reporting --------------------------------------------------------------------------- Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 - bugfix: code did not compile without zlib diff --git a/doc/v3compatibility.html b/doc/v3compatibility.html index 51619947..ad8776bb 100644 --- a/doc/v3compatibility.html +++ b/doc/v3compatibility.html @@ -95,7 +95,7 @@ set the local address the server should listen to via $UDPServerAddress

    The following example configures an UDP syslog server at the local address 192.0.2.1 on port 514:

    $ModLoad imudp
    -$UDPSeverAddress 192.0.2.1 # this MUST be before the $UDPServerRun +$UDPServerAddress 192.0.2.1 # this MUST be before the $UDPServerRun directive!
    $UDPServerRun 514

    "$UDPServerAddress *" means listen on all local interfaces. @@ -103,10 +103,10 @@ This is the default if no directive is specified.

    Please note that now multiple listeners are supported. For example, you can do the following:

    $ModLoad imudp
    -$UDPSeverAddress 192.0.2.1 # this MUST be before the $UDPServerRun +$UDPServerAddress 192.0.2.1 # this MUST be before the $UDPServerRun directive!
    $UDPServerRun 514
    -$UDPSeverAddress * # all local interfaces
    +$UDPServerAddress * # all local interfaces
    $UDPServerRun 1514

    These config file settings run two listeners: one at 192.0.2.1:514 and one on port 1514, which listens on all local -- cgit From 8b7649fcda9eb15951dcad7728220228b7e29303 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Dec 2008 14:39:22 +0100 Subject: preparing for 4.1.2 release --- ChangeLog | 4 ++-- doc/status.html | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index d60e6f76..c6bc46d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,3 @@ -- doc bugfix: typo in v3 compatibility document - thanks to Andrej for reporting --------------------------------------------------------------------------- Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 - bugfix: code did not compile without zlib @@ -12,6 +10,8 @@ Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 emitted only once in a minute (this currently is a hard-coded limit, if someone comes up with a good reason to make it configurable, we will probably do that). +- doc bugfix: typo in v3 compatibility document directive syntax + thanks to Andrej for reporting - imported other changes from 3.21.8 and 3.20.1 (see there) --------------------------------------------------------------------------- Version 4.1.1 [DEVEL] (rgerhards), 2008-11-26 diff --git a/doc/status.html b/doc/status.html index b814d0bc..74694280 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,16 +5,16 @@

    This page reflects the status as of 2008-11-26.

    Current Releases

    -

    development: 4.1.1 [2008-11-26] - -change log - -download +

    development: 4.1.2 [2008-12-05] - +change log - +download -
    beta: 3.21.7 [2008-11-11] - -change log - -download

    +
    beta: 3.21.9 [2008-12-04] - +change log - +download

    -

    v3 stable: 3.20.0 [2008-11-05] - change log - -download +

    v3 stable: 3.20.2 [2008-12-04] - change log - +download
    v2 stable: 2.0.6 [2008-08-07] - change log - download -- cgit From 128edc1598a13c894fe3853673d1231b9feafc39 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 13:31:55 +0100 Subject: bugfix: imudp went into an endless loop under some circumstances (but could also leave it under some other circumstances...) Thanks to David Lang and speedfox for reporting this issue. --- ChangeLog | 3 +++ plugins/imudp/imudp.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c6bc46d0..0ebc2eef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- bugfix: imudp went into an endless loop under some circumstances + (but could also leave it under some other circumstances...) + Thanks to David Lang and speedfox for reporting this issue. --------------------------------------------------------------------------- Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 - bugfix: code did not compile without zlib diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 037da56d..0193ac06 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -297,12 +297,12 @@ CODESTARTrunInput /* wait for io to become ready */ nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); - for (i = 0; nfds && i < *udpLstnSocks; i++) { - if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { + for(i = 0; nfds && i < *udpLstnSocks; i++) { + if(FD_ISSET(udpLstnSocks[i+1], &readfds)) { processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, fromHost, fromHostFQDN, fromHostIP); - } --nfds; /* indicate we have processed one descriptor */ + } } /* end of a run, back to loop for next recv() */ } -- cgit From 60b8ce14bf33e76237cf82dd1f68acc750e64316 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 15:42:47 +0100 Subject: added $PreserveFQDN config file directive Enables to use FQDNs in sender names where the legacy default --- ChangeLog | 3 +++ doc/rsyslog_conf_global.html | 3 +++ plugins/imudp/imudp.c | 1 + runtime/glbl.c | 5 +++++ runtime/glbl.h | 4 +++- runtime/net.c | 4 +++- runtime/parser.c | 2 +- 7 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0ebc2eef..aa53fff2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- added $PreserveFQDN config file directive + Enables to use FQDNs in sender names where the legacy default + would have stripped the domain part. - bugfix: imudp went into an endless loop under some circumstances (but could also leave it under some other circumstances...) Thanks to David Lang and speedfox for reporting this issue. diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d02245e3..6e03e571 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -186,6 +186,9 @@ supported in order to be compliant to the upcoming new syslog RFC series.

  • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common.
    • +
  • $PreserveFQDN [on/off) - if set to off (legacy default to remain compatible +to sysklogd), the domain part from a name that is within the same domain as the receiving +system is stripped. If set to on, full names are always used.
  • $WorkDirectory <name> (directory for spool and other work files)
  • $UDPServerAddress <IP> (imudp) -- local IP address (or name) the UDP listens should bind to
    • diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 0193ac06..72450513 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -181,6 +181,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* check if we have a different sender than before, if so, we need to query some new values */ if(memcmp(&frominet, frominetPrev, socklen) != 0) { CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); +DBGPRINTF("returned: fromHost '%s', FQDN: '%s'\n", fromHost, fromHostFQDN); memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ /* Here we check if a host is permitted to send us * syslog messages. If it isn't, we do not further diff --git a/runtime/glbl.c b/runtime/glbl.c index 2a6bfb11..d06c88ff 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -53,6 +53,7 @@ DEFobjStaticHelpers static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ static int bHUPisRestart = 1; /* should SIGHUP cause a full system restart? */ +static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ @@ -88,6 +89,7 @@ static dataType Get##nameFunc(void) \ } SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) +SIMP_PROP(PreserveFQDN, bPreserveFQDN, int) SIMP_PROP(HUPisRestart, bHUPisRestart, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ @@ -178,6 +180,7 @@ CODESTARTobjQueryInterface(glbl) pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(PreserveFQDN); SIMP_PROP(HUPisRestart); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); @@ -224,6 +227,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; bHUPisRestart = 1; + bPreserveFQDN = 0; return RS_RET_OK; } @@ -245,6 +249,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) diff --git a/runtime/glbl.h b/runtime/glbl.h index 44e41e3e..205a5212 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -43,6 +43,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(MaxLine, int) SIMP_PROP(OptimizeUniProc, int) SIMP_PROP(HUPisRestart, int) + SIMP_PROP(PreserveFQDN, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) @@ -57,7 +58,8 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*) #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ PROTOTYPEObj(glbl); diff --git a/runtime/net.c b/runtime/net.c index 1472b4db..30c923fe 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -1204,7 +1204,9 @@ rsRetVal cvthname(struct sockaddr_storage *f, uchar *pszHost, uchar *pszHostFQDN * make this in option in the long term. (rgerhards, 2007-09-11) */ strcpy((char*)pszHost, (char*)pszHostFQDN); - if ((p = (uchar*) strchr((char*)pszHost, '.'))) { /* find start of domain name "machine.example.com" */ + if( (glbl.GetPreserveFQDN() == 0) + && (p = (uchar*) strchr((char*)pszHost, '.'))) { /* find start of domain name "machine.example.com" */ + strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain())); if(strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()) == 0) { *p = '\0'; /* simply terminate the string */ } else { diff --git a/runtime/parser.c b/runtime/parser.c index b549cd19..b4ab0a3e 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -263,7 +263,7 @@ rsRetVal parseMsg(msg_t *pMsg) CHKiRet(sanitizeMessage(pMsg)); /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ - DBGPRINTF("msg parser: flags %x, from '%s', msg %s\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); + DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); /* pull PRI */ pri = DEFUPRI; -- cgit From 6ce531106bc26c117da8e9700d489b462e41eb8f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 19:18:55 +0100 Subject: added sponsor information - thanks to BlinkMind for sponsoring $PreserveFQDN --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index aa53fff2..efbd6afb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,8 @@ - added $PreserveFQDN config file directive Enables to use FQDNs in sender names where the legacy default would have stripped the domain part. + Thanks to BlinkMind, Inc. http://www.blinkmind.com for sponsoring this + development. - bugfix: imudp went into an endless loop under some circumstances (but could also leave it under some other circumstances...) Thanks to David Lang and speedfox for reporting this issue. -- cgit From a10bc421fffbeaa872ae0cdcb651f0a7e613ee7f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 19:55:27 +0100 Subject: resolved compile problem, e.g. on FreeBSD I commented out some debug code that is only useful in some testing scenarios and re-enabled the old code. This solved a FreeBSD compile issue. Also, I fixed some other syntax error, which somehow went into the tree (I am still puzzled about that, especially as some have already and successfully build from that tree... anyhow ;)). --- runtime/debug.c | 4 ++-- runtime/net.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/debug.c b/runtime/debug.c index 102d5d0f..53291fc2 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -819,8 +819,8 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(altdbg != -1) write(altdbg, pszWriteBuf, lenWriteBuf); } - // old, reenable TODO lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%s: ", pszThrdName); - lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "{%ld}%s: ", (long) syscall(SYS_gettid), pszThrdName); + lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%s: ", pszThrdName); + // use for testing: lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "{%ld}%s: ", (long) syscall(SYS_gettid), pszThrdName); if(stddbg != -1) write(stddbg, pszWriteBuf, lenWriteBuf); if(altdbg != -1) write(altdbg, pszWriteBuf, lenWriteBuf); /* print object name header if we have an object */ diff --git a/runtime/net.c b/runtime/net.c index 30c923fe..6fa27658 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -1206,7 +1206,7 @@ rsRetVal cvthname(struct sockaddr_storage *f, uchar *pszHost, uchar *pszHostFQDN strcpy((char*)pszHost, (char*)pszHostFQDN); if( (glbl.GetPreserveFQDN() == 0) && (p = (uchar*) strchr((char*)pszHost, '.'))) { /* find start of domain name "machine.example.com" */ - strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain())); + strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()); if(strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()) == 0) { *p = '\0'; /* simply terminate the string */ } else { -- cgit From 483be404716d8e3e55f200955e1904219eb97a9b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Dec 2008 14:26:19 +0100 Subject: enhanced imtcp, among others to handel invalid NetScreen framing - added $InputTCPServerAddtlFrameDelimiter config directive, which enabeles to specify an additional, non-standard message delimiter for processing plain tcp syslog. This is primarily a fix for the invalid framing used in Juniper's NetScreen products. Credit to forum user Arv for suggesting this solution. - added $InputTCPServerInputName property, which enables a name to be specified that will be available during message processing in the inputname property. This is considered useful for logic that treats messages differently depending on which input received them. --- ChangeLog | 9 +++++++++ doc/imtcp.html | 27 ++++++++++++++++++++++++++- plugins/imtcp/imtcp.c | 17 +++++++++++++++++ tcps_sess.c | 12 +++++++----- tcpsrv.c | 35 +++++++++++++++++++++++++++++++++++ tcpsrv.h | 19 ++++++++++++++++++- 6 files changed, 112 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index efbd6afb..f95989e2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +- added $InputTCPServerAddtlFrameDelimiter config directive, which + enabeles to specify an additional, non-standard message delimiter + for processing plain tcp syslog. This is primarily a fix for the invalid + framing used in Juniper's NetScreen products. Credit to forum user + Arv for suggesting this solution. +- added $InputTCPServerInputName property, which enables a name to be + specified that will be available during message processing in the + inputname property. This is considered useful for logic that treats + messages differently depending on which input received them. - added $PreserveFQDN config file directive Enables to use FQDNs in sender names where the legacy default would have stripped the domain part. diff --git a/doc/imtcp.html b/doc/imtcp.html index 583cd531..0ee0f96a 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -20,11 +20,36 @@ $InputTCPServerRun multiple times. This is not currently supported.

    Configuration Directives:

      +
    • $InputTCPServerAddtlFrameDelimiter <Delimiter>
      +This directive permits to specify an additional frame delimiter for plain tcp syslog. +The industry-standard specifies using the LF character as frame delimiter. Some vendors, +notable Juniper in their NetScreen products, use an invalid frame delimiter, in Juniper's +case the NUL character. This directive permits to specify the ASCII value of the delimiter +in question. Please note that this does not guarantee that all wrong implementations can +be cured with this directive. It is not even a sure fix with all versions of NetScreen, +as I suggest the NUL character is the effect of a (common) coding error and thus will +probably go away at some time in the future. But for the time being, the value 0 can +probably be used to make rsyslog handle NetScreen's invalid syslog/tcp framing. +For additional information, see this +forum thread. +
      If this doesn't work for you, please do not blame the rsyslog team. Instead file +a bug report with Juniper! +
      Note that a similar, but worse, issue exists with Cisco's IOS implementation. They do +not use any framing at all. This is confirmed from Cisco's side, but there seems to be +very limited interest in fixing this issue. This directive can not fix the Cisco bug. +That would require much more code changes, which I was unable to do so far. Full details +can be found at the Cisco tcp syslog anomaly +page.
    • $InputTCPServerRun <port>
      Starts a TCP server on selected port
      • $InputTCPMaxSessions <number>
      Sets the maximum number of sessions supported
    • $InputTCPServerStreamDriverMode <number>
      -Sets the driver mode for the currently selected network stream driver. <number> is driver specifc.
    • $InputTCPServerStreamDriverAuthMode <mode-string>
      +Sets the driver mode for the currently selected network stream driver. <number> is driver specifc.
      • +
    • $InputTCPServerInputName <name>
      +Sets a name for the inputname property. If no name is set "imtcp" is used by default. Setting a +name is not strictly necessary, but can be useful to apply filtering based on which input +the message was received from. +
    • $InputTCPServerStreamDriverAuthMode <mode-string>
      Sets the authentication mode for the currently selected network stream driver. <mode-string> is driver specifc.
    • $InputTCPServerStreamDriverPermittedPeer <id-string>
      Sets permitted peer IDs. Only these peers are able to connect to the listener. <id-string> semantics depend on the currently selected diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 89f1dbcf..19138d94 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -80,7 +80,9 @@ static permittedPeers_t *pPermPeersRoot = NULL; /* config settings */ static int iTCPSessMax = 200; /* max number of sessions */ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ +static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ +static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ /* callbacks */ @@ -166,6 +168,8 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? (uchar*)"imtcp" : pszInputName)); + CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); /* now set optional params, but only if they were actually configured */ if(pszStrmDrvrAuthMode != NULL) { CHKiRet(tcpsrv.SetDrvrAuthMode(pOurTcpsrv, pszStrmDrvrAuthMode)); @@ -239,6 +243,15 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus { iTCPSessMax = 200; iStrmDrvrMode = 0; + iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + if(pszInputName != NULL) { + free(pszInputName); + pszInputName = NULL; + } + if(pszStrmDrvrAuthMode != NULL) { + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; + } return RS_RET_OK; } @@ -273,6 +286,10 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserveraddtlframedelimiter", 0, eCmdHdlrInt, + NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverinputname", 0, + eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/tcps_sess.c b/tcps_sess.c index e8bef5b1..d0edc018 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -231,7 +231,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->bAtStrtOfFram = 1; } @@ -314,7 +314,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -323,9 +323,11 @@ processDataRcvd(tcps_sess_t *pThis, char c) */ } - if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ + if(( (c == '\n') + || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) + ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -344,7 +346,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tcpsrv.c b/tcpsrv.c index 885edba3..bb81a281 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -513,6 +513,7 @@ finalize_it: /* this is a very special case - this time only we do not exit the /* Standard-Constructor */ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ pThis->iSessMax = TCPSESS_MAX_DEFAULT; /* TODO: useful default ;) */ + pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; ENDobjConstruct(tcpsrv) @@ -560,6 +561,8 @@ CODESTARTobjDestruct(tcpsrv) free(pThis->pszDrvrAuthMode); if(pThis->ppLstn != NULL) free(pThis->ppLstn); + if(pThis->pszInputName != NULL) + free(pThis->pszInputName); ENDobjDestruct(tcpsrv) @@ -658,6 +661,36 @@ SetUsrP(tcpsrv_t *pThis, void *pUsr) } +/* Set additional framing to use (if any) -- rgerhards, 2008-12-10 */ +static rsRetVal +SetAddtlFrameDelim(tcpsrv_t *pThis, int iDelim) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->addtlFrameDelim = iDelim; + RETiRet; +} + + +/* Set the input name to use -- rgerhards, 2008-12-10 */ +static rsRetVal +SetInputName(tcpsrv_t *pThis, uchar *name) +{ + uchar *pszName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + if(name == NULL) + pszName = NULL; + else + CHKmalloc(pszName = (uchar*)strdup((char*)name)); + if(pThis->pszInputName != NULL) + free(pThis->pszInputName); + pThis->pszInputName = pszName; +finalize_it: + RETiRet; +} + + /* here follows a number of methods that shuffle authentication settings down * to the drivers. Drivers not supporting these settings may return an error * state. @@ -727,6 +760,8 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->Run = Run; pIf->SetUsrP = SetUsrP; + pIf->SetInputName = SetInputName; + pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; pIf->SetDrvrMode = SetDrvrMode; pIf->SetDrvrAuthMode = SetDrvrAuthMode; pIf->SetDrvrPermPeers = SetDrvrPermPeers; diff --git a/tcpsrv.h b/tcpsrv.h index 01110866..2924bafa 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -25,17 +25,28 @@ #include "obj.h" #include "tcps_sess.h" +/* support for framing anomalies */ +typedef enum ETCPsyslogFramingAnomaly { + frame_normal = 0, + frame_NetScreen = 1, + frame_CiscoIOS = 2 +} eTCPsyslogFramingAnomaly; + +#define TCPSRV_NO_ADDTL_DELIMITER -1 /* specifies that no additional delimiter is to be used in TCP framing */ + /* the tcpsrv object */ struct tcpsrv_s { BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ netstrms_t *pNS; /**< pointer to network stream subsystem */ int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ + uchar *pszInputName; /**< value to be used as input name */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ int iLstnMax; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ int iSessMax; /**< max number of sessions supported */ char *TCPLstnPort; /**< the port the listener shall listen on */ + int addtlFrameDelim; /**< additional frame delimiter for plain TCP syslog framing (e.g. to handle NetScreen) */ tcps_sess_t **pSessions;/**< array of all of our sessions */ void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ /* callbacks */ @@ -64,6 +75,8 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*create_tcp_socket)(tcpsrv_t *pThis); rsRetVal (*Run)(tcpsrv_t *pThis); /* set methods */ + rsRetVal (*SetAddtlFrameDelim)(tcpsrv_t*, int); + rsRetVal (*SetInputName)(tcpsrv_t*, uchar*); rsRetVal (*SetUsrP)(tcpsrv_t*, void*); rsRetVal (*SetCBIsPermittedHost)(tcpsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); rsRetVal (*SetCBOpenLstnSocks)(tcpsrv_t *, rsRetVal (*)(tcpsrv_t*)); @@ -80,7 +93,11 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetCBOnSessDestruct)(tcpsrv_t*, rsRetVal (*) (void*)); rsRetVal (*SetCBOnSessConstructFinalize)(tcpsrv_t*, rsRetVal (*) (void*)); ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +/* change for v4: + * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 + * - SetInputName() added -- rgerhards, 2008-12-10 + */ /* prototypes */ -- cgit From 246962be65941c7994aa18e4f9327f239c62eb9e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Dec 2008 11:35:58 +0100 Subject: preparing for 4.1.3 release --- ChangeLog | 4 +++- configure.ac | 2 +- doc/features.html | 2 +- doc/manual.html | 2 +- doc/status.html | 6 +++--- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index e82169fe..5056f68d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ +--------------------------------------------------------------------------- +Version 4.1.3 [DEVEL] (rgerhards), 2008-12-17 - added $InputTCPServerAddtlFrameDelimiter config directive, which - enabeles to specify an additional, non-standard message delimiter + enables to specify an additional, non-standard message delimiter for processing plain tcp syslog. This is primarily a fix for the invalid framing used in Juniper's NetScreen products. Credit to forum user Arv for suggesting this solution. diff --git a/configure.ac b/configure.ac index e8aa644a..d204b7fd 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.3],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/features.html b/doc/features.html index 7a31a03b..336b31cc 100644 --- a/doc/features.html +++ b/doc/features.html @@ -101,7 +101,7 @@ arithmetic expressions in message filters

      • World's first

      Rsyslog has an interesting number of "world's firsts" - things that were implemented for the first time ever in rsyslog. Some of them are still features not available elsewhere.
        -
      • world's first implementation of IETF I-D syslog-protocol (February 2006, version 1.12.2 and above)
      • world's first implementation of dynamic syslog on-the-wire compression (December 2006, version 1.13.0 and above)
      • world's first open-source implementation of a disk-queueing syslogd (January 2008, version 3.11.0 and above)
        • +
      • world's first implementation of IETF I-D syslog-protocol (February 2006, version 1.12.2 and above), now RFC5424
      • world's first implementation of dynamic syslog on-the-wire compression (December 2006, version 1.13.0 and above)
      • world's first open-source implementation of a disk-queueing syslogd (January 2008, version 3.11.0 and above)
      • world's first implementation of IETF I-D syslog-transport-tls (May 2008, version 3.19.0 and above)
      diff --git a/doc/manual.html b/doc/manual.html index 63a68b4f..e8842de6 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.1.2 (devel branch) of rsyslog. +

      This documentation is for version 4.1.3 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might diff --git a/doc/status.html b/doc/status.html index 74694280..c1a42963 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,9 +5,9 @@

      This page reflects the status as of 2008-11-26.

      Current Releases

      -

      development: 4.1.2 [2008-12-05] - -change log - -download +

      development: 4.1.3 [2008-12-17] - +change log - +download
      beta: 3.21.9 [2008-12-04] - change log - -- cgit From a185665be4cf699752589d81ef6e396dd61f68b6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Dec 2008 12:52:49 +0100 Subject: experimentally altered "last message repeated n times" to include msg This was suggested by David Lang, to help identify the message that was repeated. A problem is that I do not have the expanded template at hand when the "last ... times" message is generated. Spending much time on this functionality is also probably not a good thing, as the whole functionality will be overhauled (and once this is done we will not at all have the output template at hand). So the approach is to use a single field - here msg - and inlcude it as a notation of what was repeated. This is far from being perfect, but eventually good enough. I will now wait for feedback before going any further. --- action.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/action.c b/action.c index dee46f16..ab4d2f4d 100644 --- a/action.c +++ b/action.c @@ -621,9 +621,15 @@ actionWriteToAction(action_t *pAction) */ if(pAction->f_prevcount > 1) { msg_t *pMsg; +#if 0 /* old */ uchar szRepMsg[64]; snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", pAction->f_prevcount); +#else + uchar szRepMsg[1024]; + snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800]", + pAction->f_prevcount, getMSG(pAction->f_pMsg)); +#endif if((pMsg = MsgDup(pAction->f_pMsg)) == NULL) { /* it failed - nothing we can do against it... */ @@ -658,12 +664,11 @@ actionWriteToAction(action_t *pAction) dbgprintf("action not yet ready again to be executed, onceInterval %d, tCurr %d, tNext %d\n", (int) pAction->iSecsExecOnceInterval, (int) getActNow(pAction), (int) (pAction->iSecsExecOnceInterval + pAction->tLastExec)); - /* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */ pAction->tLastExec = getActNow(pAction); /* re-init time flags */ FINALIZE; } - /* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */ + /* we use reception time, not dequeue time - this is considered more appropriate and also faster ;) -- rgerhards, 2008-09-17 */ pAction->f_time = pAction->f_pMsg->ttGenTime; /* When we reach this point, we have a valid, non-disabled action. -- cgit From ec2019abe255949d5adcafc8f81cb949b45885e3 Mon Sep 17 00:00:00 2001 From: varmojfekoj Date: Mon, 12 Jan 2009 12:37:41 +0100 Subject: fixed type in format string (s as string indicator missing) Signed-off-by: Rainer Gerhards --- action.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.c b/action.c index ab4d2f4d..c95c0def 100644 --- a/action.c +++ b/action.c @@ -627,7 +627,7 @@ actionWriteToAction(action_t *pAction) pAction->f_prevcount); #else uchar szRepMsg[1024]; - snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800]", + snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", pAction->f_prevcount, getMSG(pAction->f_pMsg)); #endif -- cgit From d5360aa57436bb931c513677bc2cbdb1733a4c6b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 19 Jan 2009 11:18:39 +0100 Subject: updated project status after 3.20.3 release --- doc/status.html | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index c1a42963..01bd4479 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,7 +2,7 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2008-11-26.

      +

      This page reflects the status as of 2009-01-19.

      Current Releases

      development: 4.1.3 [2008-12-17] - @@ -13,8 +13,8 @@ change log - download

      -

      v3 stable: 3.20.2 [2008-12-04] - change log - -download +

      v3 stable: 3.20.3 [2009-01-19] - change log - +download
      v2 stable: 2.0.6 [2008-08-07] - change log - download @@ -24,7 +24,8 @@ upgrade, you may consider purchasing a out that it is really not a good idea to still run a v0 version.

      If you updgrade from version 2, be sure to read the rsyslog v3 -compatibility document.

      +compatibility document. There are no additional compatibility concerns at this time for +upgrading from v3 to v4. If some occur, we will post an additional compatiblity document.

      (How are versions named?)

      Platforms

      -- cgit From ead2c355e3261f98817ccd52bc3644103140e824 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 26 Jan 2009 22:30:55 +0100 Subject: bugfix: unitialized mutex was used in msg.c:getPRI This was subtle, because getPRI is called as part of the debugging code (always executed) in syslogd.c:logmsg. --- runtime/msg.c | 9 +++++++-- runtime/msg.h | 1 + tools/syslogd.c | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 2e2d41ad..02a4cd8a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -190,6 +190,7 @@ static void MsgPrepareEnqueueLockingCase(msg_t *pThis) * rgerhards, 2008-07-14 */ pthread_mutexattr_destroy(&pThis->mutAttr); + pThis->bDoLock = 1; ENDfunc } @@ -199,14 +200,16 @@ static void MsgLockLockingCase(msg_t *pThis) { /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ assert(pThis != NULL); - pthread_mutex_lock(&pThis->mut); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_lock(&pThis->mut); } static void MsgUnlockLockingCase(msg_t *pThis) { /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ assert(pThis != NULL); - pthread_mutex_unlock(&pThis->mut); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_unlock(&pThis->mut); } /* delete the mutex object on message destruction (locking case) @@ -745,6 +748,7 @@ char *getMSG(msg_t *pM) char *getPRI(msg_t *pM) { int pri; + BEGINfunc if(pM == NULL) return ""; @@ -764,6 +768,7 @@ char *getPRI(msg_t *pM) } MsgUnlock(pM); + ENDfunc return (char*)pM->pszPRI; } diff --git a/runtime/msg.h b/runtime/msg.h index d98111a8..c8350626 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,6 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; +short bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 2cac8fe4..f0d63932 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2431,6 +2431,7 @@ init(void) ABORT_FINALIZE(RS_RET_VALIDATION_RUN); /* switch the message object to threaded operation, if necessary */ +/* TODO:XXX: I think we must do this also if we have action queues! -- rgerhards, 2009-01-26 */ if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { MsgEnableThreadSafety(); } -- cgit From 14d5cc7f55ffc7980c0bb73f50b53da175271358 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 29 Jan 2009 11:58:26 +0100 Subject: fixed atomic operations --- runtime/atomic.h | 2 +- runtime/msg.c | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/runtime/atomic.h b/runtime/atomic.h index 7ad8e2e4..ec7acb8c 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -47,7 +47,7 @@ # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) #else -# warning "atomic builtins not available, using nul operations" +# warning "atomic builtins not available, using nul operations - rsyslogd will probably be racy!" # define ATOMIC_INC(data) (++(data)) # define ATOMIC_DEC(data) (--(data)) # define ATOMIC_DEC_AND_FETCH(data) (--(data)) diff --git a/runtime/msg.c b/runtime/msg.c index 2e2d41ad..cf291b5d 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -328,14 +328,13 @@ finalize_it: BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ int currRefCount; CODESTARTobjDestruct(msg) - /* DEV Debugging only ! dbgprintf("msgDestruct\t0x%lx, Ref now: %d\n", (unsigned long)pM, pM->iRefCount - 1); */ -//# ifdef DO_HAVE_ATOMICS -// currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); -//# else + /* DEV Debugging only ! dbgprintf("msgDestruct\t0x%lx, Ref now: %d\n", (unsigned long)pThis, pThis->iRefCount - 1); */ +# ifdef HAVE_ATOMIC_BUILTINS + currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); +# else MsgLock(pThis); currRefCount = --pThis->iRefCount; -//# endif -// we need a mutex, because we may be suspended after getting the refcount but before +# endif if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ -- cgit From 51d3552866c6b81b0860d32f8142f3bb83d0054f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 29 Jan 2009 15:20:05 +0100 Subject: preparing for 4.1.4 release --- ChangeLog | 8 ++++++++ configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1bd552b9..0d6f651d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +--------------------------------------------------------------------------- +Version 4.1.4 [DEVEL] (rgerhards), 2009-01-29 +- bugfix: inconsistent use of mutex/atomic operations could cause segfault + details are too many, for full analysis see blog post at: + http://blog.gerhards.net/2009/01/rsyslog-data-race-analysis.html +- bugfix: unitialized mutex was used in msg.c:getPRI + This was subtle, because getPRI is called as part of the debugging code + (always executed) in syslogd.c:logmsg. - bufgix: $PreserveFQDN was not properly handled for locally emitted messages --------------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index d204b7fd..1bf460ad 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.3],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.4],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/manual.html b/doc/manual.html index bc4c0bc5..bd927d78 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.1.3 (devel branch) of rsyslog. +

      This documentation is for version 4.1.4 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit From c5ece5504314797027c3742891db99759707c6b2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 29 Jan 2009 16:47:09 +0100 Subject: minor doc: updated project status --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index 01bd4479..9a6677b3 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,12 +2,12 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-01-19.

      +

      This page reflects the status as of 2009-01-29.

      Current Releases

      -

      development: 4.1.3 [2008-12-17] - -change log - -download +

      development: 4.1.4 [2009-01-29] - +change log - +download
      beta: 3.21.9 [2008-12-04] - change log - -- cgit From 94f8ca5e2cc12dd2ea9fed6027b531b182cfe73d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 30 Jan 2009 13:28:10 +0100 Subject: added useful internal doc link --- doc/rsyslog_conf_templates.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index 90b5fafe..6c68b801 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -33,7 +33,8 @@ bit restricted currently.

      All text in the template is used literally, except for things within percent signs. These are properties and allow you access to the contents of the syslog message. Properties are accessed via the -property replacer (nice name, huh) and it can do cool things, too. For +property replacer +(nice name, huh) and it can do cool things, too. For example, it can pick a substring or do date-specific formatting. More on this is below, on some lines of the property replacer.

      -- cgit From f826c284d2fc1b68ca85a237d5c6ac92a5233534 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 2 Feb 2009 14:45:03 +0100 Subject: updated project status after 3.21.10 release --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index 9a6677b3..f9e5852c 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,16 +2,16 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-01-29.

      +

      This page reflects the status as of 2009-02-02.

      Current Releases

      development: 4.1.4 [2009-01-29] - change log - download -
      beta: 3.21.9 [2008-12-04] - -change log - -download

      +
      beta: 3.21.10 [2009-02-02] - +change log - +download

      v3 stable: 3.20.3 [2009-01-19] - change log - download -- cgit From 6ce4a9d605e62b32ed62b5d6e498de5202565cee Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 2 Feb 2009 15:28:24 +0100 Subject: added new config directive $RepeatedMsgContainsOriginalMsg so that the "last message repeated n times" messages, if generated, may have an alternate format that contains the message that is being repeated. Note that this is on an action-by-action basis. --- ChangeLog | 5 +++++ action.c | 19 +++++++++++-------- action.h | 3 ++- doc/rsyslog_conf_global.html | 8 +++++++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5056f68d..acd3df4d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,9 @@ --------------------------------------------------------------------------- +Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? +- added new config directive $RepeatedMsgContainsOriginalMsg so that the + "last message repeated n times" messages, if generated, may + have an alternate format that contains the message that is being repeated +--------------------------------------------------------------------------- Version 4.1.3 [DEVEL] (rgerhards), 2008-12-17 - added $InputTCPServerAddtlFrameDelimiter config directive, which enables to specify an additional, non-standard message delimiter diff --git a/action.c b/action.c index c95c0def..a41f976c 100644 --- a/action.c +++ b/action.c @@ -58,6 +58,7 @@ static int iActExecEveryNthOccur = 0; /* execute action every n-th occurence (0, static time_t iActExecEveryNthOccurTO = 0; /* timeout for n-occurence setting (in seconds, 0=never) */ static int glbliActionResumeInterval = 30; int glbliActionResumeRetryCount = 0; /* how often should suspended actions be retried? */ +static int bActionRepMsgHasMsg = 0; /* last messsage repeated... has msg fragment in it */ /* main message queue and its configuration parameters */ static queueType_t ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ @@ -621,15 +622,7 @@ actionWriteToAction(action_t *pAction) */ if(pAction->f_prevcount > 1) { msg_t *pMsg; -#if 0 /* old */ - uchar szRepMsg[64]; - snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", - pAction->f_prevcount); -#else uchar szRepMsg[1024]; - snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", - pAction->f_prevcount, getMSG(pAction->f_pMsg)); -#endif if((pMsg = MsgDup(pAction->f_pMsg)) == NULL) { /* it failed - nothing we can do against it... */ @@ -637,6 +630,14 @@ actionWriteToAction(action_t *pAction) ABORT_FINALIZE(RS_RET_ERR); } + if(pAction->bRepMsgHasMsg == 0) { /* old format repeat message? */ + snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", + pAction->f_prevcount); + } else { + snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", + pAction->f_prevcount, getMSG(pAction->f_pMsg)); + } + /* We now need to update the other message properties. * ... RAWMSG is a problem ... Please note that digital * signatures inside the message are also invalidated. @@ -823,6 +824,7 @@ actionAddCfSysLineHdrl(void) CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinToHr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtime", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccur, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtimetimeout", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccurTO, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgcontainsoriginalmsg", 0, eCmdHdlrBinary, NULL, &bActionRepMsgHasMsg, NULL)); finalize_it: RETiRet; @@ -856,6 +858,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques pAction->iSecsExecOnceInterval = iActExecOnceInterval; pAction->iExecEveryNthOccur = iActExecEveryNthOccur; pAction->iExecEveryNthOccurTO = iActExecEveryNthOccurTO; + pAction->bRepMsgHasMsg = bActionRepMsgHasMsg; iActExecEveryNthOccur = 0; /* auto-reset */ iActExecEveryNthOccurTO = 0; /* auto-reset */ diff --git a/action.h b/action.h index 8d9d5102..e35e634c 100644 --- a/action.h +++ b/action.h @@ -57,7 +57,8 @@ struct action_s { time_t tLastOccur; /* time last occurence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ - int f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ + short bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ + short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ int iNumTpls; /* number of array entries for template element below */ diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 6e03e571..b0c1e400 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -181,6 +181,12 @@ supported in order to be compliant to the upcoming new syslog RFC series.

    • $ModDir
    • $ModLoad
      • +
    • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, +have a different format that contains the message that is being repeated. +Note that only the first "n" characters are included, with n to be at least 80 characters, most +probably more (this may change from version to version, thus no specific limit is given). The bottom +line is that n is large enough to get a good idea which message was repeated but it is not necessarily +large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.
    • $RepeatedMsgReduction
    • $ResetConfigVariables
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better @@ -226,7 +232,7 @@ point of view, "1,,0.0.,.,0" also has the value 1000.

      [rsyslog site]

      This documentation is part of the rsyslog project.
      -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL version 2 or higher.

      -- cgit From 5005bce38763051b5b12e48ac60c3ff17097a952 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 4 Mar 2009 18:22:48 +0100 Subject: added ERE support in filter conditions new comparison operation "ereregex" --- ChangeLog | 2 ++ doc/rsyslog_conf_filter.html | 8 +++++++- runtime/conf.c | 2 ++ runtime/stringbuf.c | 44 ++++++++++++++++++++++++++++++++++---------- runtime/stringbuf.h | 2 +- tools/syslogd.c | 7 ++++++- tools/syslogd.h | 3 ++- 7 files changed, 54 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index ba2a6c13..f1bc354c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? +- added ERE support in filter conditions + new comparison operation "ereregex" - added new config directive $RepeatedMsgContainsOriginalMsg so that the "last message repeated n times" messages, if generated, may have an alternate format that contains the message that is being repeated diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html index cf21ff95..1d30d8ae 100644 --- a/doc/rsyslog_conf_filter.html +++ b/doc/rsyslog_conf_filter.html @@ -141,7 +141,13 @@ once they are implemented, it can make very much sense regex Compares the property against the provided POSIX -regular +BRE regular +expression. + + +ereregex +Compares the property against the provided POSIX +ERE regular expression. diff --git a/runtime/conf.c b/runtime/conf.c index f71d5669..c5208d86 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -873,6 +873,8 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) f->f_filterData.prop.operation = FIOP_STARTSWITH; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "regex", 5)) { f->f_filterData.prop.operation = FIOP_REGEX; + } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "ereregex", 8)) { + f->f_filterData.prop.operation = FIOP_EREREGEX; } else { errmsg.LogError(0, NO_ERRCODE, "error: invalid compare operation '%s' - ignoring selector", (char*) rsCStrGetSzStrNoNULL(pCSCompOp)); diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 93d1e1ef..150439a1 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -694,6 +694,32 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) return -1; /* pCS1 is less then psz */ } +#if 0 +/* check if a CStr object matches a POSIX ERE regex. + * added 2009-03-04 by rgerhards + * TODO: we should merge this with rsCStrSzStrMatchReg + */ +rsRetVal rsCStrSzStrMatchRegexERE(cstr_t *pCS1, uchar *psz) +{ + regex_t preq; + DEFiRet; + + BEGINfunc + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), REG_EXTENDED); + ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); + regexp.regfree(&preq); + } else { + ret = 1; /* simulate "not found" */ + } + + ENDfunc + RETiRet; +} +#endif + + /* check if a CStr object matches a regex. * msamia@redhat.com 2007-07-12 * @return returns 0 if matched @@ -701,25 +727,23 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) * rgerhards, 2007-07-16: bug is no real bug, because rsyslogd ensures there * never is a \0 *inside* a property string. * Note that the function returns -1 if regexp functionality is not available. - * TODO: change calling interface! -- rgerhards, 2008-03-07 + * rgerhards: 2009-03-04: ERE support added, via parameter iType: 0 - BRE, 1 - ERE */ -int rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz) +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) { regex_t preq; - int ret; - - BEGINfunc + DEFiRet; if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), 0); - ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); + regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), iType == 1 ? REG_EXTENDED : 0); + CHKiRet(regexp.regexec(&preq, (char*) psz, 0, NULL, 0)); regexp.regfree(&preq); } else { - ret = 1; /* simulate "not found" */ + ABORT_FINALIZE(RS_RET_NOT_FOUND); } - ENDfunc - return ret; +finalize_it: + RETiRet; } diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index c1966449..f3e08439 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -136,7 +136,7 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz); int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); -int rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz); +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); diff --git a/tools/syslogd.c b/tools/syslogd.c index 9ced4562..6b8ce82f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1071,7 +1071,12 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce break; case FIOP_REGEX: if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal) == 0) + (unsigned char*) pszPropVal, 0) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 1) == RS_RET_OK) bRet = 1; break; default: diff --git a/tools/syslogd.h b/tools/syslogd.h index e866a16b..f1b11a91 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -70,7 +70,8 @@ struct filed { FIOP_CONTAINS = 1, /* contains string? */ FIOP_ISEQUAL = 2, /* is (exactly) equal? */ FIOP_STARTSWITH = 3, /* starts with a string? */ - FIOP_REGEX = 4 /* matches a regular expression? */ + FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ + FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } operation; cstr_t *pCSCompValue; /* value to "compare" against */ char isNegated; /* actually a boolean ;) */ -- cgit From bbcbd87ebdafe8a344c375104264a9ebdd3948d5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 4 Mar 2009 18:34:36 +0100 Subject: some optimization on regex code also some commented-out leftover removed --- runtime/stringbuf.c | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 150439a1..8d52834d 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -694,31 +694,6 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) return -1; /* pCS1 is less then psz */ } -#if 0 -/* check if a CStr object matches a POSIX ERE regex. - * added 2009-03-04 by rgerhards - * TODO: we should merge this with rsCStrSzStrMatchReg - */ -rsRetVal rsCStrSzStrMatchRegexERE(cstr_t *pCS1, uchar *psz) -{ - regex_t preq; - DEFiRet; - - BEGINfunc - - if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), REG_EXTENDED); - ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); - regexp.regfree(&preq); - } else { - ret = 1; /* simulate "not found" */ - } - - ENDfunc - RETiRet; -} -#endif - /* check if a CStr object matches a regex. * msamia@redhat.com 2007-07-12 @@ -735,7 +710,7 @@ rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) DEFiRet; if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), iType == 1 ? REG_EXTENDED : 0); + regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); CHKiRet(regexp.regexec(&preq, (char*) psz, 0, NULL, 0)); regexp.regfree(&preq); } else { -- cgit From 42db7de5968d2db0fa855a9f029f6bccc0a30650 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 4 Mar 2009 18:46:06 +0100 Subject: fixed newly introduced memory leak (bug created 30 minutes ago or so) --- runtime/stringbuf.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 8d52834d..a5dc625a 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -707,12 +707,15 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) { regex_t preq; + int ret; DEFiRet; if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); - CHKiRet(regexp.regexec(&preq, (char*) psz, 0, NULL, 0)); + ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); regexp.regfree(&preq); + if(ret != 0) + ABORT_FINALIZE(RS_RET_NOT_FOUND); } else { ABORT_FINALIZE(RS_RET_NOT_FOUND); } -- cgit From 2e388db9ac91eae35ac836b329c8bcadd319a409 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 5 Mar 2009 11:10:43 +0100 Subject: integrated various patches for solaris Unfortunatley, I do not have the full list of contributors available. The patch set was compiled by Ben Taylor, and I made some further changes to adopt it to the news rsyslog branch. Others provided much of the base work, but I can not find the names of the original authors. If you happen to be one of them, please let me know so that I can give proper credits. --- action.c | 44 +++--- action.h | 2 +- configure.ac | 12 +- runtime/conf.c | 7 +- runtime/modules.c | 4 + runtime/net.c | 5 + runtime/queue.c | 420 +++++++++++++++++++++++++++--------------------------- runtime/queue.h | 50 +++---- runtime/rsyslog.c | 2 +- runtime/srutils.c | 2 +- runtime/wti.c | 5 + runtime/wtp.c | 5 + tools/Makefile.am | 2 +- tools/omfile.c | 4 + tools/syslogd.c | 63 ++++---- 15 files changed, 335 insertions(+), 292 deletions(-) diff --git a/action.c b/action.c index a41f976c..cd4ba240 100644 --- a/action.c +++ b/action.c @@ -180,7 +180,7 @@ rsRetVal actionDestruct(action_t *pThis) ASSERT(pThis != NULL); if(pThis->pQueue != NULL) { - queueDestruct(&pThis->pQueue); + qqueueDestruct(&pThis->pQueue); } if(pThis->pMod != NULL) @@ -255,7 +255,7 @@ actionConstructFinalize(action_t *pThis) * to be run on multiple threads. So far, this is forbidden by the interface * spec. -- rgerhards, 2008-01-30 */ - CHKiRet(queueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, (rsRetVal (*)(void*,void*))actionCallDoAction)); + CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, (rsRetVal (*)(void*,void*))actionCallDoAction)); obj.SetName((obj_t*) pThis->pQueue, pszQName); /* ... set some properties ... */ @@ -268,24 +268,24 @@ actionConstructFinalize(action_t *pThis) errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ } - queueSetpUsr(pThis->pQueue, pThis); - setQPROP(queueSetsizeOnDiskMax, "$ActionQueueMaxDiskSpace", iActionQueMaxDiskSpace); - setQPROP(queueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); - setQPROPstr(queueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); - setQPROP(queueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); - setQPROP(queueSettoQShutdown, "$ActionQueueTimeoutShutdown", iActionQtoQShutdown ); - setQPROP(queueSettoActShutdown, "$ActionQueueTimeoutActionCompletion", iActionQtoActShutdown); - setQPROP(queueSettoWrkShutdown, "$ActionQueueWorkerTimeoutThreadShutdown", iActionQtoWrkShutdown); - setQPROP(queueSettoEnq, "$ActionQueueTimeoutEnqueue", iActionQtoEnq); - setQPROP(queueSetiHighWtrMrk, "$ActionQueueHighWaterMark", iActionQHighWtrMark); - setQPROP(queueSetiLowWtrMrk, "$ActionQueueLowWaterMark", iActionQLowWtrMark); - setQPROP(queueSetiDiscardMrk, "$ActionQueueDiscardMark", iActionQDiscardMark); - setQPROP(queueSetiDiscardSeverity, "$ActionQueueDiscardSeverity", iActionQDiscardSeverity); - setQPROP(queueSetiMinMsgsPerWrkr, "$ActionQueueWorkerThreadMinimumMessages", iActionQWrkMinMsgs); - setQPROP(queueSetbSaveOnShutdown, "$ActionQueueSaveOnShutdown", bActionQSaveOnShutdown); - setQPROP(queueSetiDeqSlowdown, "$ActionQueueDequeueSlowdown", iActionQueueDeqSlowdown); - setQPROP(queueSetiDeqtWinFromHr, "$ActionQueueDequeueTimeBegin", iActionQueueDeqtWinFromHr); - setQPROP(queueSetiDeqtWinToHr, "$ActionQueueDequeueTimeEnd", iActionQueueDeqtWinToHr); + qqueueSetpUsr(pThis->pQueue, pThis); + setQPROP(qqueueSetsizeOnDiskMax, "$ActionQueueMaxDiskSpace", iActionQueMaxDiskSpace); + setQPROP(qqueueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); + setQPROPstr(qqueueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); + setQPROP(qqueueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); + setQPROP(qqueueSettoQShutdown, "$ActionQueueTimeoutShutdown", iActionQtoQShutdown ); + setQPROP(qqueueSettoActShutdown, "$ActionQueueTimeoutActionCompletion", iActionQtoActShutdown); + setQPROP(qqueueSettoWrkShutdown, "$ActionQueueWorkerTimeoutThreadShutdown", iActionQtoWrkShutdown); + setQPROP(qqueueSettoEnq, "$ActionQueueTimeoutEnqueue", iActionQtoEnq); + setQPROP(qqueueSetiHighWtrMrk, "$ActionQueueHighWaterMark", iActionQHighWtrMark); + setQPROP(qqueueSetiLowWtrMrk, "$ActionQueueLowWaterMark", iActionQLowWtrMark); + setQPROP(qqueueSetiDiscardMrk, "$ActionQueueDiscardMark", iActionQDiscardMark); + setQPROP(qqueueSetiDiscardSeverity, "$ActionQueueDiscardSeverity", iActionQDiscardSeverity); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$ActionQueueWorkerThreadMinimumMessages", iActionQWrkMinMsgs); + setQPROP(qqueueSetbSaveOnShutdown, "$ActionQueueSaveOnShutdown", bActionQSaveOnShutdown); + setQPROP(qqueueSetiDeqSlowdown, "$ActionQueueDequeueSlowdown", iActionQueueDeqSlowdown); + setQPROP(qqueueSetiDeqtWinFromHr, "$ActionQueueDequeueTimeBegin", iActionQueueDeqtWinFromHr); + setQPROP(qqueueSetiDeqtWinToHr, "$ActionQueueDequeueTimeEnd", iActionQueueDeqtWinToHr); # undef setQPROP # undef setQPROPstr @@ -294,7 +294,7 @@ actionConstructFinalize(action_t *pThis) bActionQSaveOnShutdown, iActionQueMaxDiskSpace); - CHKiRet(queueStart(pThis->pQueue)); + CHKiRet(qqueueStart(pThis->pQueue)); dbgprintf("Action %p: queue %p created\n", pThis, pThis->pQueue); /* and now reset the queue params (see comment in its function header!) */ @@ -675,7 +675,7 @@ actionWriteToAction(action_t *pAction) /* When we reach this point, we have a valid, non-disabled action. * So let's enqueue our message for execution. -- rgerhards, 2007-07-24 */ - iRet = queueEnqObj(pAction->pQueue, pAction->f_pMsg->flowCtlType, (void*) MsgAddRef(pAction->f_pMsg)); + iRet = qqueueEnqObj(pAction->pQueue, pAction->f_pMsg->flowCtlType, (void*) MsgAddRef(pAction->f_pMsg)); if(iRet == RS_RET_OK) pAction->f_prevcount = 0; /* message processed, so we start a new cycle */ diff --git a/action.h b/action.h index e35e634c..dc9bbd74 100644 --- a/action.h +++ b/action.h @@ -68,7 +68,7 @@ struct action_s { * content later). This is preserved after the message has been * processed - it is also used to detect duplicates. */ - queue_t *pQueue; /* action queue */ + qqueue_t *pQueue; /* action queue */ SYNC_OBJ_TOOL; /* required for mutex support */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ }; diff --git a/configure.ac b/configure.ac index 0c924754..ff790817 100644 --- a/configure.ac +++ b/configure.ac @@ -35,6 +35,13 @@ case "${host}" in # do not DEFINE OS_BSD os_type="bsd" ;; + *-*-solaris*) + os_type="solaris" + AC_DEFINE([OS_SOLARIS], [1], [Indicator for a Solaris OS]) + AC_DEFINE([_POSIX_PTHREAD_SEMANTICS], [1], [Use POSIX pthread semantics]) + SOL_LIBS="-lsocket -lnsl" + AC_SUBST(SOL_LIBS) + ;; esac AC_DEFINE_UNQUOTED([HOSTENV], "$host", [the host environment, can be queried via a system variable]) @@ -233,7 +240,10 @@ if test "x$enable_pthreads" != "xno"; then [ AC_DEFINE([USE_PTHREADS], [1], [Multithreading support enabled.]) PTHREADS_LIBS="-lpthread" - PTHREADS_CFLAGS="-pthread" + case "${os_type}" in + solaris) PTHREADS_CFLAGS="-pthreads" ;; + *) PTHREADS_CFLAGS="-pthread" ;; + esac AC_SUBST(PTHREADS_LIBS) AC_SUBST(PTHREADS_CFLAGS) ], diff --git a/runtime/conf.c b/runtime/conf.c index c5208d86..a670c65b 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -46,7 +46,9 @@ #include #include #ifdef HAVE_LIBGEN_H -# include +# ifndef OS_SOLARIS +# include +# endif #endif #include "rsyslog.h" @@ -68,6 +70,9 @@ #include "ctok.h" #include "ctok_token.h" +#ifdef OS_SOLARIS +# define NAME_MAX MAXNAMELEN +#endif /* forward definitions */ static rsRetVal cfline(uchar *line, selector_t **pfCurr); diff --git a/runtime/modules.c b/runtime/modules.c index 169d234b..d548a949 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -49,6 +49,10 @@ #include #include +#ifdef OS_SOLARIS +# define PATH_MAX MAXPATHLEN +#endif + #include "cfsysline.h" #include "modules.h" #include "errmsg.h" diff --git a/runtime/net.c b/runtime/net.c index 4e6d54a1..db2d7e37 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -63,6 +63,11 @@ #include "errmsg.h" #include "net.h" +#ifdef OS_SOLARIS +# define s6_addr32 _S6_un._S6_u32 + typedef unsigned int u_int32_t; +#endif + MODULE_TYPE_LIB /* static data */ diff --git a/runtime/queue.c b/runtime/queue.c index a3e29a96..7d78460c 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -56,14 +56,14 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) /* forward-definitions */ -rsRetVal queueChkPersist(queue_t *pThis); -static rsRetVal queueSetEnqOnly(queue_t *pThis, int bEnqOnly, int bLockMutex); -static rsRetVal queueRateLimiter(queue_t *pThis); -static int queueChkStopWrkrDA(queue_t *pThis); -static int queueIsIdleDA(queue_t *pThis); -static rsRetVal queueConsumerDA(queue_t *pThis, wti_t *pWti, int iCancelStateSave); -static rsRetVal queueConsumerCancelCleanup(void *arg1, void *arg2); -static rsRetVal queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex); +rsRetVal qqueueChkPersist(qqueue_t *pThis); +static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); +static rsRetVal qqueueRateLimiter(qqueue_t *pThis); +static int qqueueChkStopWrkrDA(qqueue_t *pThis); +static int qqueueIsIdleDA(qqueue_t *pThis); +static rsRetVal qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); +static rsRetVal qqueueConsumerCancelCleanup(void *arg1, void *arg2); +static rsRetVal qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -77,7 +77,7 @@ static rsRetVal queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex); * rgerhards, 2008-01-29 */ static inline int -queueGetOverallQueueSize(queue_t *pThis) +qqueueGetOverallQueueSize(qqueue_t *pThis) { #if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */ BEGINfunc @@ -96,7 +96,7 @@ ENDfunc * This function returns void, as it makes no sense to communicate an error back, even if * it happens. */ -static inline void queueDrain(queue_t *pThis) +static inline void queueDrain(qqueue_t *pThis) { void *pUsr; @@ -119,26 +119,26 @@ static inline void queueDrain(queue_t *pThis) * this point in time. The mutex must be locked when * ths function is called. -- rgerhards, 2008-01-25 */ -static inline rsRetVal queueAdviseMaxWorkers(queue_t *pThis) +static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) { DEFiRet; int iMaxWorkers; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(!pThis->bEnqOnly) { if(pThis->bRunsDA) { /* if we have not yet reached the high water mark, there is no need to start a * worker. -- rgerhards, 2008-01-26 */ - if(queueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { + if(qqueueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } } else { if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { iMaxWorkers = 1; } else { - iMaxWorkers = queueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + iMaxWorkers = qqueueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ } @@ -153,11 +153,11 @@ static inline rsRetVal queueAdviseMaxWorkers(queue_t *pThis) * rgerhards, 2008-02-27 */ static rsRetVal -queueWaitDAModeInitialized(queue_t *pThis) +qqueueWaitDAModeInitialized(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->bRunsDA); while(pThis->bRunsDA != 2) { @@ -179,17 +179,17 @@ queueWaitDAModeInitialized(queue_t *pThis) * rgerhards, 2008-01-15 */ static rsRetVal -queueTurnOffDAMode(queue_t *pThis) +qqueueTurnOffDAMode(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->bRunsDA); /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need * to wait for its startup... -- rgerhards, 2008-01-25 */ - queueWaitDAModeInitialized(pThis); + qqueueWaitDAModeInitialized(pThis); /* if we need to pull any data that we still need from the (child) disk queue, * now would be the time to do so. At present, we do not need this, but I'd like to @@ -208,15 +208,15 @@ queueTurnOffDAMode(queue_t *pThis) /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, * this will be quick. */ - queueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ + qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ dbgoprint((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n", iRet); /* now we need to check if the regular queue has some messages. This may be the case * when it is waiting that the high water mark is reached again. If so, we need to start up * a regular worker. -- rgerhards, 2008-01-26 */ - if(queueGetOverallQueueSize(pThis) > 0) { - queueAdviseMaxWorkers(pThis); + if(qqueueGetOverallQueueSize(pThis) > 0) { + qqueueAdviseMaxWorkers(pThis); } } @@ -232,11 +232,11 @@ queueTurnOffDAMode(queue_t *pThis) * rgerhards, 2008-01-14 */ static rsRetVal -queueChkIsDA(queue_t *pThis) +qqueueChkIsDA(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pszFilePrefix != NULL) { pThis->bIsDA = 1; dbgoprint((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); @@ -260,18 +260,18 @@ queueChkIsDA(queue_t *pThis) * rgerhards, 2008-01-15 */ static rsRetVal -queueStartDA(queue_t *pThis) +qqueueStartDA(qqueue_t *pThis) { DEFiRet; uchar pszDAQName[128]; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->bRunsDA == 2) /* check if already in (fully initialized) DA mode... */ FINALIZE; /* ... then we are already done! */ /* create message queue */ - CHKiRet(queueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); + CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); /* give it a name */ snprintf((char*) pszDAQName, sizeof(pszDAQName)/sizeof(uchar), "%s[DA]", obj.GetName((obj_t*) pThis)); @@ -282,30 +282,30 @@ queueStartDA(queue_t *pThis) */ pThis->pqDA->pqParent = pThis; - CHKiRet(queueSetpUsr(pThis->pqDA, pThis->pUsr)); - CHKiRet(queueSetsizeOnDiskMax(pThis->pqDA, pThis->sizeOnDiskMax)); - CHKiRet(queueSetiDeqSlowdown(pThis->pqDA, pThis->iDeqSlowdown)); - CHKiRet(queueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); - CHKiRet(queueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(queueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); - CHKiRet(queueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); - CHKiRet(queueSettoEnq(pThis->pqDA, pThis->toEnq)); - CHKiRet(queueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); - CHKiRet(queueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); - CHKiRet(queueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); - CHKiRet(queueSetiHighWtrMrk(pThis->pqDA, 0)); - CHKiRet(queueSetiDiscardMrk(pThis->pqDA, 0)); + CHKiRet(qqueueSetpUsr(pThis->pqDA, pThis->pUsr)); + CHKiRet(qqueueSetsizeOnDiskMax(pThis->pqDA, pThis->sizeOnDiskMax)); + CHKiRet(qqueueSetiDeqSlowdown(pThis->pqDA, pThis->iDeqSlowdown)); + CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); + CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); + CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); + CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); + CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); + CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); + CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); + CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); + CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0)); if(pThis->toQShutdown == 0) { - CHKiRet(queueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ } else { /* we use the shortest possible shutdown (0 is endless!) because when we run on disk AND * have an obviously large backlog, we can't finish it in any case. So there is no point * in holding shutdown longer than necessary. -- rgerhards, 2008-01-15 */ - CHKiRet(queueSettoQShutdown(pThis->pqDA, 1)); + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 1)); } - iRet = queueStart(pThis->pqDA); + iRet = qqueueStart(pThis->pqDA); /* file not found is expected, that means it is no previous QIF available */ if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) FINALIZE; /* something is wrong */ @@ -323,12 +323,12 @@ queueStartDA(queue_t *pThis) pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */ dbgoprint((obj_t*) pThis, "is now running in disk assisted mode, disk queue 0x%lx\n", - queueGetID(pThis->pqDA)); + qqueueGetID(pThis->pqDA)); finalize_it: if(iRet != RS_RET_OK) { if(pThis->pqDA != NULL) { - queueDestruct(&pThis->pqDA); + qqueueDestruct(&pThis->pqDA); } dbgoprint((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); pThis->bIsDA = 0; @@ -345,7 +345,7 @@ finalize_it: * rgerhards, 2008-01-16 */ static inline rsRetVal -queueInitDA(queue_t *pThis, int bEnqOnly, int bLockMutex) +qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -363,12 +363,12 @@ queueInitDA(queue_t *pThis, int bEnqOnly, int bLockMutex) lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DA", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpDA)); CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) queueChkStopWrkrDA)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) queueIsIdleDA)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) queueConsumerDA)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) queueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) queueStartDA)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) queueTurnOffDAMode)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); + CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerDA)); + CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) qqueueConsumerCancelCleanup)); + CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); + CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); @@ -401,14 +401,14 @@ finalize_it: * rgerhards, 2008-01-14 */ static inline rsRetVal -queueChkStrtDA(queue_t *pThis) +qqueueChkStrtDA(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* if we do not hit the high water mark, we have nothing to do */ - if(queueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) + if(qqueueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) ABORT_FINALIZE(RS_RET_OK); if(pThis->bRunsDA) { @@ -422,15 +422,15 @@ queueChkStrtDA(queue_t *pThis) * we need at least one). */ dbgoprint((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n", - queueGetOverallQueueSize(pThis)); - queueAdviseMaxWorkers(pThis); + qqueueGetOverallQueueSize(pThis)); + qqueueAdviseMaxWorkers(pThis); } else { /* this is the case when we are currently not running in DA mode. So it is time * to turn it back on. */ dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", - queueGetOverallQueueSize(pThis)); - queueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ + qqueueGetOverallQueueSize(pThis)); + qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ } finalize_it: @@ -448,7 +448,7 @@ finalize_it: */ /* -------------------- fixed array -------------------- */ -static rsRetVal qConstructFixedArray(queue_t *pThis) +static rsRetVal qConstructFixedArray(qqueue_t *pThis) { DEFiRet; @@ -464,14 +464,14 @@ static rsRetVal qConstructFixedArray(queue_t *pThis) pThis->tVars.farray.head = 0; pThis->tVars.farray.tail = 0; - queueChkIsDA(pThis); + qqueueChkIsDA(pThis); finalize_it: RETiRet; } -static rsRetVal qDestructFixedArray(queue_t *pThis) +static rsRetVal qDestructFixedArray(qqueue_t *pThis) { DEFiRet; @@ -486,7 +486,7 @@ static rsRetVal qDestructFixedArray(queue_t *pThis) } -static rsRetVal qAddFixedArray(queue_t *pThis, void* in) +static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) { DEFiRet; @@ -499,7 +499,7 @@ static rsRetVal qAddFixedArray(queue_t *pThis, void* in) RETiRet; } -static rsRetVal qDelFixedArray(queue_t *pThis, void **out) +static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out) { DEFiRet; @@ -518,7 +518,7 @@ static rsRetVal qDelFixedArray(queue_t *pThis, void **out) /* first some generic functions which are also used for the unget linked list */ -static inline rsRetVal queueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) +static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) { DEFiRet; qLinkedList_t *pEntry; @@ -544,7 +544,7 @@ finalize_it: RETiRet; } -static inline rsRetVal queueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) +static inline rsRetVal qqueueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) { DEFiRet; qLinkedList_t *pEntry; @@ -571,7 +571,7 @@ static inline rsRetVal queueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t /* end generic functions which are also used for the unget linked list */ -static rsRetVal qConstructLinkedList(queue_t *pThis) +static rsRetVal qConstructLinkedList(qqueue_t *pThis) { DEFiRet; @@ -580,13 +580,13 @@ static rsRetVal qConstructLinkedList(queue_t *pThis) pThis->tVars.linklist.pRoot = 0; pThis->tVars.linklist.pLast = 0; - queueChkIsDA(pThis); + qqueueChkIsDA(pThis); RETiRet; } -static rsRetVal qDestructLinkedList(queue_t __attribute__((unused)) *pThis) +static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) { DEFiRet; @@ -599,11 +599,11 @@ static rsRetVal qDestructLinkedList(queue_t __attribute__((unused)) *pThis) RETiRet; } -static rsRetVal qAddLinkedList(queue_t *pThis, void* pUsr) +static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) { DEFiRet; - iRet = queueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); + iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); #if 0 qLinkedList_t *pEntry; @@ -627,10 +627,10 @@ finalize_it: RETiRet; } -static rsRetVal qDelLinkedList(queue_t *pThis, obj_t **ppUsr) +static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr) { DEFiRet; - iRet = queueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); + iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); #if 0 qLinkedList_t *pEntry; @@ -657,11 +657,11 @@ static rsRetVal qDelLinkedList(queue_t *pThis, obj_t **ppUsr) static rsRetVal -queueLoadPersStrmInfoFixup(strm_t *pStrm, queue_t __attribute__((unused)) *pThis) +qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pThis) { DEFiRet; ISOBJ_TYPE_assert(pStrm, strm); - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); CHKiRet(strmSetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); finalize_it: RETiRet; @@ -673,14 +673,14 @@ finalize_it: * rgerhards, 2008-01-15 */ static rsRetVal -queueHaveQIF(queue_t *pThis) +qqueueHaveQIF(qqueue_t *pThis) { DEFiRet; uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; struct stat stat_buf; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pszFilePrefix == NULL) ABORT_FINALIZE(RS_RET_NO_FILEPREFIX); @@ -710,7 +710,7 @@ finalize_it: * rgerhards, 2008-01-11 */ static rsRetVal -queueTryLoadPersistedInfo(queue_t *pThis) +qqueueTryLoadPersistedInfo(qqueue_t *pThis) { DEFiRet; strm_t *psQIF = NULL; @@ -720,7 +720,7 @@ queueTryLoadPersistedInfo(queue_t *pThis) int iUngottenObjs; obj_t *pUsr; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", @@ -755,15 +755,15 @@ queueTryLoadPersistedInfo(queue_t *pThis) while(iUngottenObjs > 0) { /* fill the queue from disk */ CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL)); - queueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); + qqueueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); --iUngottenObjs; /* one less */ } /* and now the stream objects (some order as when persisted!) */ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF, - (rsRetVal(*)(obj_t*,void*))queueLoadPersStrmInfoFixup, pThis)); + (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, - (rsRetVal(*)(obj_t*,void*))queueLoadPersStrmInfoFixup, pThis)); + (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pRead)); @@ -793,7 +793,7 @@ finalize_it: * allowed file size at this point - that should be a config setting... * rgerhards, 2008-01-10 */ -static rsRetVal qConstructDisk(queue_t *pThis) +static rsRetVal qConstructDisk(qqueue_t *pThis) { DEFiRet; int bRestarted = 0; @@ -801,7 +801,7 @@ static rsRetVal qConstructDisk(queue_t *pThis) ASSERT(pThis != NULL); /* and now check if there is some persistent information that needs to be read in */ - iRet = queueTryLoadPersistedInfo(pThis); + iRet = qqueueTryLoadPersistedInfo(pThis); if(iRet == RS_RET_OK) bRestarted = 1; else if(iRet != RS_RET_FILE_NOT_FOUND) @@ -843,7 +843,7 @@ finalize_it: } -static rsRetVal qDestructDisk(queue_t *pThis) +static rsRetVal qDestructDisk(qqueue_t *pThis) { DEFiRet; @@ -855,7 +855,7 @@ static rsRetVal qDestructDisk(queue_t *pThis) RETiRet; } -static rsRetVal qAddDisk(queue_t *pThis, void* pUsr) +static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) { DEFiRet; number_t nWriteCount; @@ -882,7 +882,7 @@ finalize_it: RETiRet; } -static rsRetVal qDelDisk(queue_t *pThis, void **ppUsr) +static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) { DEFiRet; @@ -913,18 +913,18 @@ finalize_it: } /* -------------------- direct (no queueing) -------------------- */ -static rsRetVal qConstructDirect(queue_t __attribute__((unused)) *pThis) +static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { return RS_RET_OK; } -static rsRetVal qDestructDirect(queue_t __attribute__((unused)) *pThis) +static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) { return RS_RET_OK; } -static rsRetVal qAddDirect(queue_t *pThis, void* pUsr) +static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { DEFiRet; @@ -941,7 +941,7 @@ static rsRetVal qAddDirect(queue_t *pThis, void* pUsr) RETiRet; } -static rsRetVal qDelDirect(queue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) +static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) { return RS_RET_OK; } @@ -956,12 +956,12 @@ static rsRetVal qDelDirect(queue_t __attribute__((unused)) *pThis, __attribute__ * rgerhards, 2008-01-20 */ static rsRetVal -queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex) +qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); /* TODO: we aborted right at this place at least 3 times -- race? 2008-02-28, -03-10, -03-15 The second time I noticed it the queue was in destruction with NO worker threads running. The pUsr ptr was totally off and provided no clue what it may be pointing @@ -970,7 +970,7 @@ queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex) dbgoprint((obj_t*) pThis, "ungetting user object %s\n", obj.GetName(pUsr)); BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); - iRet = queueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); + iRet = qqueueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); ++pThis->iUngottenObjs; /* indicate one more */ END_MTX_PROTECTED_OPERATIONS(pThis->mut); @@ -986,14 +986,14 @@ queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex) * rgerhards, 2008-01-29 */ static rsRetVal -queueGetUngottenObj(queue_t *pThis, obj_t **ppUsr) +qqueueGetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(ppUsr != NULL); - iRet = queueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); + iRet = qqueueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); --pThis->iUngottenObjs; /* indicate one less */ dbgoprint((obj_t*) pThis, "dequeued ungotten user object %s\n", obj.GetName(*ppUsr)); @@ -1007,7 +1007,7 @@ queueGetUngottenObj(queue_t *pThis, obj_t **ppUsr) * things truely different. -- rgerhards, 2008-02-12 */ static rsRetVal -queueAdd(queue_t *pThis, void *pUsr) +qqueueAdd(qqueue_t *pThis, void *pUsr) { DEFiRet; @@ -1030,7 +1030,7 @@ finalize_it: * ungotten list and, if so, dequeue it first. */ static rsRetVal -queueDel(queue_t *pThis, void *pUsr) +qqueueDel(qqueue_t *pThis, void *pUsr) { DEFiRet; @@ -1042,7 +1042,7 @@ queueDel(queue_t *pThis, void *pUsr) * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ if(pThis->iUngottenObjs > 0) { - iRet = queueGetUngottenObj(pThis, (obj_t**) pUsr); + iRet = qqueueGetUngottenObj(pThis, (obj_t**) pUsr); } else { iRet = pThis->qDel(pThis, pUsr); ATOMIC_DEC(pThis->iQueueSize); @@ -1066,14 +1066,14 @@ queueDel(queue_t *pThis, void *pUsr) * complex) if each would have its own shutdown. The function does not self check * this condition - the caller must make sure it is not called with a parent. */ -static rsRetVal queueShutdownWorkers(queue_t *pThis) +static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) { DEFiRet; DEFVARS_mutexProtection; struct timespec tTimeout; rsRetVal iRetLocal; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); @@ -1087,7 +1087,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) /* first try to shutdown the queue within the regular shutdown period */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(queueGetOverallQueueSize(pThis) > 0) { + if(qqueueGetOverallQueueSize(pThis) > 0) { if(pThis->bRunsDA) { /* We may have waited on the low water mark. As it may have changed, we * see if we reactivate the worker. @@ -1125,7 +1125,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) if(pThis->bRunsDA) { END_MTX_PROTECTED_OPERATIONS(pThis->mut); dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", - queueGetID(pThis->pqDA)); + qqueueGetID(pThis->pqDA)); /* we use the same absolute timeout as above, so we do not use more than the configured * timeout interval! */ @@ -1154,19 +1154,19 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ if(pThis->bRunsDA) - queueWaitDAModeInitialized(pThis); + qqueueWaitDAModeInitialized(pThis); BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ /* optimize parameters for shutdown of DA-enabled queues */ - if(pThis->bIsDA && queueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + if(pThis->bIsDA && qqueueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { /* switch to enqueue-only mode so that no more actions happen */ if(pThis->bRunsDA == 0) { - queueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ + qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ } else { /* TODO: RACE: we may reach this point when the DA worker has been initialized (state 1) * but is not yet running (state 2). In this case, pThis->pqDA is NULL! rgerhards, 2008-02-27 */ - queueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ + qqueueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ } END_MTX_PROTECTED_OPERATIONS(pThis->mut); /* make sure we do not timeout before we are done */ @@ -1188,7 +1188,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) * they will automatically terminate as there no longer is any message left to process. */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(queueGetOverallQueueSize(pThis) > 0) { + if(qqueueGetOverallQueueSize(pThis) > 0) { timeoutComp(&tTimeout, pThis->toActShutdown); if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { END_MTX_PROTECTED_OPERATIONS(pThis->mut); @@ -1257,7 +1257,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers * may still be running. */ - dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", queueGetOverallQueueSize(pThis)); + dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", qqueueGetOverallQueueSize(pThis)); RETiRet; } @@ -1269,17 +1269,17 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) * is done by queueStart(). The reason is that we want to give the caller a chance * to modify some parameters before the queue is actually started. */ -rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, +rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)) { DEFiRet; - queue_t *pThis; + qqueue_t *pThis; ASSERT(ppThis != NULL); ASSERT(pConsumer != NULL); ASSERT(iWorkerThreads >= 0); - if((pThis = (queue_t *)calloc(1, sizeof(queue_t))) == NULL) { + if((pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -1316,7 +1316,7 @@ rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, pThis->qConstruct = qConstructLinkedList; pThis->qDestruct = qDestructLinkedList; pThis->qAdd = qAddLinkedList; - pThis->qDel = (rsRetVal (*)(queue_t*,void**)) qDelLinkedList; + pThis->qDel = (rsRetVal (*)(qqueue_t*,void**)) qDelLinkedList; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; @@ -1343,25 +1343,25 @@ finalize_it: /* cancellation cleanup handler for queueWorker () * Updates admin structure and frees ressources. * Params: - * arg1 - user pointer (in this case a queue_t) + * arg1 - user pointer (in this case a qqueue_t) * arg2 - user data pointer (in this case a queue data element, any object [queue's pUsr ptr!]) * Note that arg2 may be NULL, in which case no dequeued but unprocessed pUsr exists! * rgerhards, 2008-01-16 */ static rsRetVal -queueConsumerCancelCleanup(void *arg1, void *arg2) +qqueueConsumerCancelCleanup(void *arg1, void *arg2) { DEFiRet; - queue_t *pThis = (queue_t*) arg1; + qqueue_t *pThis = (qqueue_t*) arg1; obj_t *pUsr = (obj_t*) arg2; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pUsr != NULL) { /* make sure the data element is not lost */ dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n"); - CHKiRet(queueUngetObj(pThis, pUsr, LOCK_MUTEX)); + CHKiRet(qqueueUngetObj(pThis, pUsr, LOCK_MUTEX)); } finalize_it: @@ -1383,13 +1383,13 @@ finalize_it: * the return state! * rgerhards, 2008-01-24 */ -static int queueChkDiscardMsg(queue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) +static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) { DEFiRet; rsRetVal iRetLocal; int iSeverity; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk && bRunsDA == 0) { @@ -1414,7 +1414,7 @@ finalize_it: * rgerhards, 2008-10-21 */ static rsRetVal -queueDequeueConsumable(queue_t *pThis, wti_t *pWti, int iCancelStateSave) +qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; void *pUsr; @@ -1422,9 +1422,9 @@ queueDequeueConsumable(queue_t *pThis, wti_t *pWti, int iCancelStateSave) int bRunsDA; /* cache for early mutex release */ /* dequeue element (still protected from mutex) */ - iRet = queueDel(pThis, &pUsr); - queueChkPersist(pThis); - iQueueSize = queueGetOverallQueueSize(pThis); /* cache this for after mutex release */ + iRet = qqueueDel(pThis, &pUsr); + qqueueChkPersist(pThis); + iQueueSize = qqueueGetOverallQueueSize(pThis); /* cache this for after mutex release */ bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ /* We now need to save the user pointer for the cancel cleanup handler, BUT ONLY @@ -1475,7 +1475,7 @@ queueDequeueConsumable(queue_t *pThis, wti_t *pWti, int iCancelStateSave) * provide real-time creation of spool files. * Note: It is OK to use the cached iQueueSize here, because it does not hurt if it is slightly wrong. */ - CHKiRet(queueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); finalize_it: if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) { @@ -1524,7 +1524,7 @@ finalize_it: * but you get the idea from the code above. */ static rsRetVal -queueRateLimiter(queue_t *pThis) +qqueueRateLimiter(qqueue_t *pThis) { DEFiRet; int iDelay; @@ -1532,7 +1532,7 @@ queueRateLimiter(queue_t *pThis) time_t tCurr; struct tm m; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ @@ -1587,14 +1587,14 @@ queueRateLimiter(queue_t *pThis) * rgerhards, 2008-01-21 */ static rsRetVal -queueConsumerReg(queue_t *pThis, wti_t *pWti, int iCancelStateSave) +qqueueConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(queueDequeueConsumable(pThis, pWti, iCancelStateSave)); + CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->pUsrp)); /* we now need to check if we should deliberately delay processing a bit @@ -1621,15 +1621,15 @@ finalize_it: * rgerhards, 2008-01-14 */ static rsRetVal -queueConsumerDA(queue_t *pThis, wti_t *pWti, int iCancelStateSave) +qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(queueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(queueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); + CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); finalize_it: dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); @@ -1645,7 +1645,7 @@ finalize_it: * the DA queue */ static int -queueChkStopWrkrDA(queue_t *pThis) +qqueueChkStopWrkrDA(qqueue_t *pThis) { /* if our queue is in destruction, we drain to the DA queue and so we shall not terminate * until we have done so. @@ -1664,7 +1664,7 @@ queueChkStopWrkrDA(queue_t *pThis) && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { /* this queue can never grow, so we can give up... */ bStopWrkr = 1; - } else if(queueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { + } else if(qqueueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { bStopWrkr = 1; } else { bStopWrkr = 0; @@ -1687,9 +1687,9 @@ queueChkStopWrkrDA(queue_t *pThis) * the DA queue */ static int -queueChkStopWrkrReg(queue_t *pThis) +qqueueChkStopWrkrReg(qqueue_t *pThis) { - return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && queueGetOverallQueueSize(pThis) == 0); + return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0); } @@ -1697,26 +1697,26 @@ queueChkStopWrkrReg(queue_t *pThis) * are not stable! DA queue version */ static int -queueIsIdleDA(queue_t *pThis) +qqueueIsIdleDA(qqueue_t *pThis) { /* remember: iQueueSize is the DA queue size, not the main queue! */ /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(queueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && queueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); } /* must only be called when the queue mutex is locked, else results * are not stable! Regular queue version */ static int -queueIsIdleReg(queue_t *pThis) +qqueueIsIdleReg(qqueue_t *pThis) { #if 0 /* enable for performance testing */ int ret; - ret = queueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && queueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); + ret = qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); if(ret) fprintf(stderr, "queue is idle\n"); return ret; #else /* regular code! */ - return(queueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && queueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); #endif } @@ -1735,11 +1735,11 @@ queueIsIdleReg(queue_t *pThis) * I am telling this, because I, too, always get confused by those... */ static rsRetVal -queueRegOnWrkrShutdown(queue_t *pThis) +qqueueRegOnWrkrShutdown(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pqParent != NULL) { pThis->pqParent->bChildIsDone = 1; /* indicate we are done */ @@ -1756,11 +1756,11 @@ queueRegOnWrkrShutdown(queue_t *pThis) * hook to indicate in the parent queue (if we are a child) that we are not done yet. */ static rsRetVal -queueRegOnWrkrStartup(queue_t *pThis) +qqueueRegOnWrkrStartup(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pqParent != NULL) { pThis->pqParent->bChildIsDone = 0; @@ -1773,7 +1773,7 @@ queueRegOnWrkrStartup(queue_t *pThis) /* start up the queue - it must have been constructed and parameters defined * before. */ -rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ +rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; rsRetVal iRetLocal; @@ -1811,7 +1811,7 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, - queueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1); + qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ @@ -1822,13 +1822,13 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpReg)); CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); - CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) queueRateLimiter)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) queueChkStopWrkrReg)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) queueIsIdleReg)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) queueConsumerReg)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))queueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) queueRegOnWrkrStartup)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) queueRegOnWrkrShutdown)); + CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRateLimiter)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrReg)); + CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleReg)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerReg)); + CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))qqueueConsumerCancelCleanup)); + CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrStartup)); + CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrShutdown)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads)); @@ -1841,10 +1841,10 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ /* If we are disk-assisted, we need to check if there is a QIF file * which we need to load. -- rgerhards, 2008-01-15 */ - iRetLocal = queueHaveQIF(pThis); + iRetLocal = qqueueHaveQIF(pThis); if(iRetLocal == RS_RET_OK) { dbgoprint((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n"); - queueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ + qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ bInitialized = 1; /* we are done */ } else { /* TODO: use logerror? -- rgerhards, 2008-01-16 */ @@ -1861,7 +1861,7 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ /* if the queue already contains data, we need to start the correct number of worker threads. This can be * the case when a disk queue has been loaded. If we did not start it here, it would never start. */ - queueAdviseMaxWorkers(pThis); + qqueueAdviseMaxWorkers(pThis); pThis->bQueueStarted = 1; finalize_it: @@ -1876,7 +1876,7 @@ finalize_it: * and 0 otherwise. * rgerhards, 2008-01-10 */ -static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) +static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) { DEFiRet; strm_t *psQIF = NULL; /* Queue Info File */ @@ -1887,7 +1887,7 @@ static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) ASSERT(pThis != NULL); if(pThis->qType != QUEUETYPE_DISK) { - if(queueGetOverallQueueSize(pThis) > 0) { + if(qqueueGetOverallQueueSize(pThis) > 0) { /* This error code is OK, but we will probably not implement this any time * The reason is that persistence happens via DA queues. But I would like to * leave the code as is, as we so have a hook in case we need one. @@ -1898,13 +1898,13 @@ static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) FINALIZE; /* if the queue is empty, we are happy and done... */ } - dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", queueGetOverallQueueSize(pThis)); + dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", qqueueGetOverallQueueSize(pThis)); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - if((bIsCheckpoint != QUEUE_CHECKPOINT) && (queueGetOverallQueueSize(pThis) == 0)) { + if((bIsCheckpoint != QUEUE_CHECKPOINT) && (qqueueGetOverallQueueSize(pThis) == 0)) { if(pThis->bNeedDelQIF) { unlink((char*)pszQIFNam); pThis->bNeedDelQIF = 0; @@ -1938,7 +1938,7 @@ static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) * to the regular files. -- rgerhards, 2008-01-29 */ while(pThis->iUngottenObjs > 0) { - CHKiRet(queueGetUngottenObj(pThis, &pUsr)); + CHKiRet(qqueueGetUngottenObj(pThis, &pUsr)); CHKiRet((objSerialize(pUsr))(pUsr, psQIF)); objDestruct(pUsr); } @@ -1972,14 +1972,14 @@ finalize_it: * abide to our regular call interface)... * rgerhards, 2008-01-13 */ -rsRetVal queueChkPersist(queue_t *pThis) +rsRetVal qqueueChkPersist(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { - queuePersist(pThis, QUEUE_CHECKPOINT); + qqueuePersist(pThis, QUEUE_CHECKPOINT); pThis->iUpdsSincePersist = 0; } @@ -1988,8 +1988,8 @@ rsRetVal queueChkPersist(queue_t *pThis) /* destructor for the queue object */ -BEGINobjDestruct(queue) /* be sure to specify the object type also in END and CODESTART macros! */ -CODESTARTobjDestruct(queue) +BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(qqueue) pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ /* shut down all workers (handles *all* of the persistence logic) @@ -1999,7 +1999,7 @@ CODESTARTobjDestruct(queue) * with a child! -- rgerhards, 2008-01-28 */ if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) - queueShutdownWorkers(pThis); + qqueueShutdownWorkers(pThis); /* finally destruct our (regular) worker thread pool * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, @@ -2024,7 +2024,7 @@ CODESTARTobjDestruct(queue) wtpDestruct(&pThis->pWtpDA); } if(pThis->pqDA != NULL) { - queueDestruct(&pThis->pqDA); + qqueueDestruct(&pThis->pqDA); } /* persist the queue (we always do that - queuePersits() does cleanup if the queue is empty) @@ -2034,7 +2034,7 @@ CODESTARTobjDestruct(queue) * disk queues and DA mode. Anyhow, it doesn't hurt to know that we could extend it here * if need arises (what I doubt...) -- rgerhards, 2008-01-25 */ - CHKiRet_Hdlr(queuePersist(pThis, QUEUE_NO_CHECKPOINT)) { + CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { dbgoprint((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); } @@ -2059,7 +2059,7 @@ CODESTARTobjDestruct(queue) if(pThis->pszSpoolDir != NULL) free(pThis->pszSpoolDir); -ENDobjDestruct(queue) +ENDobjDestruct(qqueue) /* set the queue's file prefix @@ -2068,7 +2068,7 @@ ENDobjDestruct(queue) * rgerhards, 2008-01-09 */ rsRetVal -queueSetFilePrefix(queue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) +qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) { DEFiRet; @@ -2091,11 +2091,11 @@ finalize_it: * rgerhards, 2008-01-09 */ rsRetVal -queueSetMaxFileSize(queue_t *pThis, size_t iMaxFileSize) +qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(iMaxFileSize < 1024) { ABORT_FINALIZE(RS_RET_VALUE_TOO_LOW); @@ -2112,13 +2112,13 @@ finalize_it: * Enqueues the new element and awakes worker thread. */ rsRetVal -queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) +qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) { DEFiRet; int iCancelStateSave; struct timespec t; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* first check if we need to discard this message (which will cause CHKiRet() to exit) * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize @@ -2127,7 +2127,7 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) * threading tools may point this access to be an error, but this is done * intentional. I do not see this causes problems to us. */ - CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); /* Please note that this function is not cancel-safe and consequently * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE @@ -2142,7 +2142,7 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* then check if we need to add an assistance disk queue */ if(pThis->bIsDA) - CHKiRet(queueChkStrtDA(pThis)); + CHKiRet(qqueueChkStrtDA(pThis)); /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. @@ -2195,13 +2195,13 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) } /* and finally enqueue the message */ - CHKiRet(queueAdd(pThis, pUsr)); - queueChkPersist(pThis); + CHKiRet(qqueueAdd(pThis, pUsr)); + qqueueChkPersist(pThis); finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { /* make sure at least one worker is running. */ - queueAdviseMaxWorkers(pThis); + qqueueAdviseMaxWorkers(pThis); /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); @@ -2228,12 +2228,12 @@ finalize_it: * rgerhards, 2008-01-16 */ static rsRetVal -queueSetEnqOnly(queue_t *pThis, int bEnqOnly, int bLockMutex) +qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* for simplicity, we do one big mutex lock. This method is extremely seldom * called, so that doesn't matter... -- rgerhards, 2008-01-16 @@ -2272,24 +2272,24 @@ finalize_it: /* some simple object access methods */ -DEFpropSetMeth(queue, iPersistUpdCnt, int) -DEFpropSetMeth(queue, iDeqtWinFromHr, int) -DEFpropSetMeth(queue, iDeqtWinToHr, int) -DEFpropSetMeth(queue, toQShutdown, long) -DEFpropSetMeth(queue, toActShutdown, long) -DEFpropSetMeth(queue, toWrkShutdown, long) -DEFpropSetMeth(queue, toEnq, long) -DEFpropSetMeth(queue, iHighWtrMrk, int) -DEFpropSetMeth(queue, iLowWtrMrk, int) -DEFpropSetMeth(queue, iDiscardMrk, int) -DEFpropSetMeth(queue, iFullDlyMrk, int) -DEFpropSetMeth(queue, iDiscardSeverity, int) -DEFpropSetMeth(queue, bIsDA, int) -DEFpropSetMeth(queue, iMinMsgsPerWrkr, int) -DEFpropSetMeth(queue, bSaveOnShutdown, int) -DEFpropSetMeth(queue, pUsr, void*) -DEFpropSetMeth(queue, iDeqSlowdown, int) -DEFpropSetMeth(queue, sizeOnDiskMax, int64) +DEFpropSetMeth(qqueue, iPersistUpdCnt, int) +DEFpropSetMeth(qqueue, iDeqtWinFromHr, int) +DEFpropSetMeth(qqueue, iDeqtWinToHr, int) +DEFpropSetMeth(qqueue, toQShutdown, long) +DEFpropSetMeth(qqueue, toActShutdown, long) +DEFpropSetMeth(qqueue, toWrkShutdown, long) +DEFpropSetMeth(qqueue, toEnq, long) +DEFpropSetMeth(qqueue, iHighWtrMrk, int) +DEFpropSetMeth(qqueue, iLowWtrMrk, int) +DEFpropSetMeth(qqueue, iDiscardMrk, int) +DEFpropSetMeth(qqueue, iFullDlyMrk, int) +DEFpropSetMeth(qqueue, iDiscardSeverity, int) +DEFpropSetMeth(qqueue, bIsDA, int) +DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int) +DEFpropSetMeth(qqueue, bSaveOnShutdown, int) +DEFpropSetMeth(qqueue, pUsr, void*) +DEFpropSetMeth(qqueue, iDeqSlowdown, int) +DEFpropSetMeth(qqueue, sizeOnDiskMax, int64) /* This function can be used as a generic way to set properties. Only the subset @@ -2298,11 +2298,11 @@ DEFpropSetMeth(queue, sizeOnDiskMax, int64) * rgerhards, 2008-01-11 */ #define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) -static rsRetVal queueSetProperty(queue_t *pThis, var_t *pProp) +static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pProp != NULL); if(isProp("iQueueSize")) { @@ -2324,19 +2324,19 @@ finalize_it: #undef isProp /* dummy */ -rsRetVal queueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } +rsRetVal qqueueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } /* Initialize the stream class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-01-09 */ -BEGINObjClassInit(queue, 1, OBJ_IS_CORE_MODULE) +BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); /* now set our own handlers */ - OBJSetMethodHandler(objMethod_SETPROPERTY, queueSetProperty); -ENDObjClassInit(queue) + OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); +ENDObjClassInit(qqueue) /* vi:set ai: */ diff --git a/runtime/queue.h b/runtime/queue.h index a2dd594f..a267862d 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -160,7 +160,7 @@ typedef struct queue_s { strm_t *pRead; /* current file to be read */ } disk; } tVars; -} queue_t; +} qqueue_t; /* some symbolic constants for easier reference */ #define QUEUE_MODE_ENQDEQ 0 @@ -177,30 +177,30 @@ typedef struct queue_s { #define QUEUE_TIMEOUT_ETERNAL 24 * 60 * 60 * 1000 /* prototypes */ -rsRetVal queueDestruct(queue_t **ppThis); -rsRetVal queueEnqObj(queue_t *pThis, flowControl_t flwCtlType, void *pUsr); -rsRetVal queueStart(queue_t *pThis); -rsRetVal queueSetMaxFileSize(queue_t *pThis, size_t iMaxFileSize); -rsRetVal queueSetFilePrefix(queue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); -rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, +rsRetVal qqueueDestruct(qqueue_t **ppThis); +rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); +rsRetVal qqueueStart(qqueue_t *pThis); +rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); +rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); +rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); -PROTOTYPEObjClassInit(queue); -PROTOTYPEpropSetMeth(queue, iPersistUpdCnt, int); -PROTOTYPEpropSetMeth(queue, iDeqtWinFromHr, int); -PROTOTYPEpropSetMeth(queue, iDeqtWinToHr, int); -PROTOTYPEpropSetMeth(queue, toQShutdown, long); -PROTOTYPEpropSetMeth(queue, toActShutdown, long); -PROTOTYPEpropSetMeth(queue, toWrkShutdown, long); -PROTOTYPEpropSetMeth(queue, toEnq, long); -PROTOTYPEpropSetMeth(queue, iHighWtrMrk, int); -PROTOTYPEpropSetMeth(queue, iLowWtrMrk, int); -PROTOTYPEpropSetMeth(queue, iDiscardMrk, int); -PROTOTYPEpropSetMeth(queue, iDiscardSeverity, int); -PROTOTYPEpropSetMeth(queue, iMinMsgsPerWrkr, int); -PROTOTYPEpropSetMeth(queue, bSaveOnShutdown, int); -PROTOTYPEpropSetMeth(queue, pUsr, void*); -PROTOTYPEpropSetMeth(queue, iDeqSlowdown, int); -PROTOTYPEpropSetMeth(queue, sizeOnDiskMax, int64); -#define queueGetID(pThis) ((unsigned long) pThis) +PROTOTYPEObjClassInit(qqueue); +PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); +PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); +PROTOTYPEpropSetMeth(qqueue, iDeqtWinToHr, int); +PROTOTYPEpropSetMeth(qqueue, toQShutdown, long); +PROTOTYPEpropSetMeth(qqueue, toActShutdown, long); +PROTOTYPEpropSetMeth(qqueue, toWrkShutdown, long); +PROTOTYPEpropSetMeth(qqueue, toEnq, long); +PROTOTYPEpropSetMeth(qqueue, iHighWtrMrk, int); +PROTOTYPEpropSetMeth(qqueue, iLowWtrMrk, int); +PROTOTYPEpropSetMeth(qqueue, iDiscardMrk, int); +PROTOTYPEpropSetMeth(qqueue, iDiscardSeverity, int); +PROTOTYPEpropSetMeth(qqueue, iMinMsgsPerWrkr, int); +PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int); +PROTOTYPEpropSetMeth(qqueue, pUsr, void*); +PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int); +PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); +#define qqueueGetID(pThis) ((unsigned long) pThis) #endif /* #ifndef QUEUE_H_INCLUDED */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 54db12c2..8df100a1 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -157,7 +157,7 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) if(ppErrObj != NULL) *ppErrObj = "wtp"; CHKiRet(wtpClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "queue"; - CHKiRet(queueClassInit(NULL)); + CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmstk"; CHKiRet(vmstkClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "sysvar"; diff --git a/runtime/srutils.c b/runtime/srutils.c index 1280e40d..d01ca20d 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -458,7 +458,7 @@ srSleep(int iSeconds, int iuSeconds) * Added 2008-01-30 */ char *rs_strerror_r(int errnum, char *buf, size_t buflen) { -#ifdef __hpux +#ifndef HAVE_STRERROR_R char *pszErr; pszErr = strerror(errnum); snprintf(buf, buflen, "%s", pszErr); diff --git a/runtime/wti.c b/runtime/wti.c index e8a77474..9b3450e6 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -39,6 +39,11 @@ #include #include +#ifdef OS_SOLARIS +# include +# define pthread_yield() sched_yield() +#endif + #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" diff --git a/runtime/wtp.c b/runtime/wtp.c index 06173e04..41903576 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -40,6 +40,11 @@ #include #include +#ifdef OS_SOLARIS +# include +# define pthread_yield() sched_yield() +#endif + #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" diff --git a/tools/Makefile.am b/tools/Makefile.am index 776279a5..e523b854 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -22,7 +22,7 @@ rsyslogd_SOURCES = \ \ ../dirty.h rsyslogd_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) +rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS diff --git a/tools/omfile.c b/tools/omfile.c index 1539ae19..ea91d6ef 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -44,6 +44,10 @@ #include #include +#ifdef OS_SOLARIS +# include +#endif + #include "syslogd.h" #include "syslogd-types.h" #include "srUtils.h" diff --git a/tools/syslogd.c b/tools/syslogd.c index 6b8ce82f..63c4b249 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -72,13 +72,18 @@ #include #include #include -#include -#ifdef __sun +#ifdef OS_SOLARIS # include +# include +# include +# include +# include #else +# include # include #endif + #include #include #include @@ -279,7 +284,7 @@ static int gidDropPriv = 0; /* group-id to which priveleges should be dropped to extern int errno; /* main message queue and its configuration parameters */ -static queue_t *pMsgQueue = NULL; /* the main message queue */ +static qqueue_t *pMsgQueue = NULL; /* the main message queue */ static int iMainMsgQueueSize = 10000; /* size of the main message queue above */ static int iMainMsgQHighWtrMark = 8000; /* high water mark for disk-assisted queues */ static int iMainMsgQLowWtrMark = 2000; /* low water mark for disk-assisted queues */ @@ -1620,7 +1625,7 @@ submitMsg(msg_t *pMsg) ISOBJ_TYPE_assert(pMsg, msg); MsgPrepareEnqueue(pMsg); - queueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); + qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); RETiRet; } @@ -1681,7 +1686,7 @@ logmsg(msg_t *pMsg, int flags) /* now submit the message to the main queue - then we are done */ pMsg->msgFlags = flags; MsgPrepareEnqueue(pMsg); - queueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); + qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); ENDfunc } @@ -1979,7 +1984,7 @@ die(int sig) /* drain queue (if configured so) and stop main queue worker thread pool */ dbgprintf("Terminating main queue...\n"); - queueDestruct(&pMsgQueue); + qqueueDestruct(&pMsgQueue); pMsgQueue = NULL; /* Free ressources and close connections. This includes flushing any remaining @@ -2269,8 +2274,8 @@ static void dbgPrintInitInfo(void) static int iMainMsgQtoWrkMinMsgs = 100; static int iMainMsgQbSaveOnShutdown = 1; iMainMsgQueMaxDiskSpace = 0; - setQPROP(queueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); - setQPROP(queueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); + setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); } @@ -2332,7 +2337,7 @@ init(void) /* delete the message queue, which also flushes all messages left over */ if(pMsgQueue != NULL) { dbgprintf("deleting main message queue\n"); - queueDestruct(&pMsgQueue); /* delete pThis here! */ + qqueueDestruct(&pMsgQueue); /* delete pThis here! */ pMsgQueue = NULL; } @@ -2444,7 +2449,7 @@ init(void) } /* create message queue */ - CHKiRet_Hdlr(queueConstruct(&pMsgQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { + CHKiRet_Hdlr(qqueueConstruct(&pMsgQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); exit(1); @@ -2462,29 +2467,29 @@ init(void) errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ } - setQPROP(queueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); - setQPROP(queueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); - setQPROPstr(queueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); - setQPROP(queueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); - setQPROP(queueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); - setQPROP(queueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); - setQPROP(queueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); - setQPROP(queueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); - setQPROP(queueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); - setQPROP(queueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); - setQPROP(queueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); - setQPROP(queueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); - setQPROP(queueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); - setQPROP(queueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); - setQPROP(queueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); - setQPROP(queueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); - setQPROP(queueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); + setQPROP(qqueueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); + setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); + setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); + setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); + setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); + setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); + setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); + setQPROP(qqueueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); + setQPROP(qqueueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); + setQPROP(qqueueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); + setQPROP(qqueueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); + setQPROP(qqueueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); + setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); + setQPROP(qqueueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); + setQPROP(qqueueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); + setQPROP(qqueueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); # undef setQPROP # undef setQPROPstr /* ... and finally start the queue! */ - CHKiRet_Hdlr(queueStart(pMsgQueue)) { + CHKiRet_Hdlr(qqueueStart(pMsgQueue)) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not start message queue - rsyslogd can not run!\n", iRet); exit(1); @@ -3093,7 +3098,7 @@ GlobalClassExit(void) CHKiRet(strmClassInit(NULL)); CHKiRet(wtiClassInit(NULL)); CHKiRet(wtpClassInit(NULL)); - CHKiRet(queueClassInit(NULL)); + CHKiRet(qqueueClassInit(NULL)); CHKiRet(vmstkClassInit(NULL)); CHKiRet(sysvarClassInit(NULL)); CHKiRet(vmClassInit(NULL)); -- cgit From f67cf99ee5cd88bda499aa52d6008bb7d4afe483 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 6 Mar 2009 14:27:15 +0100 Subject: fixed a platform issue the prevented building on solaris --- runtime/queue.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/queue.c b/runtime/queue.c index 7d78460c..c4a0fad2 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -51,6 +51,11 @@ #include "wti.h" #include "atomic.h" +#ifdef OS_SOLARIS +# include +# define pthread_yield() sched_yield() +#endif + /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) -- cgit From e8499c6d33d09f6d8b42df72da1661be0ef0f088 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 10 Mar 2009 17:37:13 +0100 Subject: initial implementation of RainerScript functions & strlen() - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet supported - but we now have a good foundation to do that later on. NOTE: nested function calls are not yet supported due to a design issue with the function call VM instruction set design. - implemented the strlen() RainerScript function --- ChangeLog | 7 +++++++ runtime/conf.c | 4 ++++ runtime/ctok.c | 17 +++++++++++---- runtime/expr.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- runtime/rsyslog.h | 4 ++++ runtime/vm.c | 28 +++++++++++++++++++++++++ runtime/vmop.c | 3 +++ runtime/vmop.h | 33 ++++++++++++++++++++++++++--- 8 files changed, 149 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index f1bc354c..7602bd41 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +- implemented function support in RainerScript. That means the engine + parses and compile functions, as well as executes a few build-in + ones. Dynamic loading and registration of functions is not yet + supported - but we now have a good foundation to do that later on. + NOTE: nested function calls are not yet supported due to a design + issue with the function call VM instruction set design. +- implemented the strlen() RainerScript function --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? - added ERE support in filter conditions diff --git a/runtime/conf.c b/runtime/conf.c index a670c65b..ede15cc7 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -794,6 +794,10 @@ dbgprintf("calling expression parser, pp %p ('%s')\n", *pline, *pline); CHKiRet(ctok.Getpp(tok, pline)); CHKiRet(ctok.Destruct(&tok)); + /* debug support - print vmprg after construction (uncomment to use) */ + /* vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); */ + vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); + /* we now need to skip whitespace to the action part, else we confuse * the legacy rsyslog conf parser. -- rgerhards, 2008-02-25 */ diff --git a/runtime/ctok.c b/runtime/ctok.c index ceab15bd..d2cd8bbd 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -259,12 +259,17 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(rsCStrConstruct(&pstrVal)); - /* this loop is quite simple, a variable name is terminated by whitespace. */ - while(!isspace(c)) { + /* this loop is quite simple, a variable name is terminated when a non-supported + * character is detected. Note that we currently permit a numerical digit as the + * first char, which is not permitted by ABNF. -- rgerhards, 2009-03-10 + */ + while(isalpha(c) || isdigit(c) || (c == '_') || (c == '-')) { CHKiRet(rsCStrAppendChar(pstrVal, tolower(c))); CHKiRet(ctokGetCharFromStream(pThis, &c)); } - CHKiRet(rsCStrFinish(pStrB)); + CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ + + CHKiRet(rsCStrFinish(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -389,6 +394,7 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) uchar c; uchar szWord[128]; int bRetry = 0; /* retry parse? Only needed for inline comments... */ + cstr_t *pstrVal; ISOBJ_TYPE_assert(pThis, ctok); ASSERT(ppToken != NULL); @@ -512,7 +518,10 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) /* push c back, higher level parser needs it */ CHKiRet(ctokUngetCharFromStream(pThis, c)); pToken->tok = ctok_FUNCTION; - /* TODO: fill function name */ + /* fill function name */ + CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(rsCStrSetSzStr(pstrVal, szWord)); + CHKiRet(var.SetString(pToken->pVar, pstrVal)); } else { /* give up... */ dbgprintf("parser has an invalid word (token) '%s'\n", szWord); pToken->tok = ctok_INVALID; diff --git a/runtime/expr.c b/runtime/expr.c index ee5b9e2c..38ed1c68 100644 --- a/runtime/expr.c +++ b/runtime/expr.c @@ -55,10 +55,62 @@ DEFobjCurrIf(ctok) * rgerhards, 2008-02-19 */ -/* forward definiton - thanks to recursive ABNF, we can not avoid at least one ;) */ +/* forward definition - thanks to recursive ABNF, we can not avoid at least one ;) */ static rsRetVal expr(expr_t *pThis, ctok_t *tok); +static rsRetVal +function(expr_t *pThis, ctok_t *tok) +{ + DEFiRet; + ctok_token_t *pToken = NULL; + int iNumArgs = 0; + var_t *pVar; + + ISOBJ_TYPE_assert(pThis, expr); + ISOBJ_TYPE_assert(tok, ctok); + + CHKiRet(ctok.GetToken(tok, &pToken)); + /* note: pToken is destructed in finalize_it */ + + if(pToken->tok == ctok_LPAREN) { + CHKiRet(ctok_token.Destruct(&pToken)); /* token processed, "eat" it */ + CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one */ + } else + ABORT_FINALIZE(RS_RET_FUNC_NO_LPAREN); + + /* we first push all the params on the stack. Then we call the function */ + while(pToken->tok != ctok_RPAREN) { + ++iNumArgs; + CHKiRet(ctok.UngetToken(tok, pToken)); /* not for us, so let others process it */ + CHKiRet(expr(pThis, tok)); + CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one, needed for while() check */ + if(pToken->tok == ctok_COMMA) { + CHKiRet(ctok_token.Destruct(&pToken)); /* token processed, "eat" it */ + CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one */ + if(pToken->tok == ctok_RPAREN) { + ABORT_FINALIZE(RS_RET_FUNC_MISSING_EXPR); + } + } + } + + + /* now push number of arguments - this must be on top of the stack */ + CHKiRet(var.Construct(&pVar)); + CHKiRet(var.ConstructFinalize(pVar)); + CHKiRet(var.SetNumber(pVar, iNumArgs)); + CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHCONSTANT, pVar)); /* add to program */ + + +finalize_it: + if(pToken != NULL) { + ctok_token.Destruct(&pToken); /* "eat" processed token */ + } + + RETiRet; +} + + static rsRetVal terminal(expr_t *pThis, ctok_t *tok) { @@ -85,8 +137,12 @@ terminal(expr_t *pThis, ctok_t *tok) break; case ctok_FUNCTION: dbgoprint((obj_t*) pThis, "function\n"); - /* TODO: vm: call - well, need to implement that first */ - ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); + CHKiRet(function(pThis, tok)); /* this creates the stack call frame */ + /* ... but we place the call instruction onto the stack ourselfs (because + * we have all relevant information) + */ + CHKiRet(ctok_token.UnlinkVar(pToken, &pVar)); + CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_FUNC_CALL, pVar)); /* add to program */ break; case ctok_MSGVAR: dbgoprint((obj_t*) pThis, "MSGVAR\n"); @@ -406,6 +462,7 @@ ENDobjQueryInterface(expr) */ BEGINObjClassInit(expr, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(vmprg, CORE_COMPONENT)); CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(ctok_token, CORE_COMPONENT)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 00290ee5..899f5e13 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -254,6 +254,10 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_TIME = -2107, /**< invalid timestamp (e.g. could not be parsed) */ RS_RET_NO_ZIP = -2108, /**< ZIP functionality is not present */ RS_RET_CODE_ERR = -2109, /**< program code (internal) error */ + RS_RET_FUNC_NO_LPAREN = -2110, /**< left parenthesis missing after function call (rainerscript) */ + RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ + RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ + RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/vm.c b/runtime/vm.c index bc6c3dd2..113a9d59 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -331,6 +331,33 @@ finalize_it: ENDop(PUSHSYSVAR) +/* The function call operation is only very roughly implemented. While the plumbing + * to reach this instruction is fine, the instruction itself currently supports only + * functions with a single argument AND with a name that we know. + * TODO: later, we can add here the real logic, that involves looking up function + * names, loading them dynamically ... and all that... + * implementation begun 2009-03-10 by rgerhards + */ +BEGINop(FUNC_CALL) /* remember to set the instruction also in the ENDop macro! */ + var_t *numOperands; + var_t *operand1; + int iStrlen; +CODESTARTop(FUNC_CALL) + vmstk.PopNumber(pThis->pStk, &numOperands); + if(numOperands->val.num != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + vmstk.PopString(pThis->pStk, &operand1); /* guess there's just one ;) */ + if(!rsCStrSzStrCmp(pOp->operand.pVar->val.pStr, (uchar*) "strlen", 6)) { /* only one supported so far ;) */ +RUNLOG_VAR("%s", rsCStrGetSzStr(operand1->val.pStr)); + iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); +RUNLOG_VAR("%d", iStrlen); + } else + ABORT_FINALIZE(RS_RET_INVLD_FUNC); + PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME +finalize_it: +ENDop(FUNC_CALL) + + /* ------------------------------ end instruction set implementation ------------------------------ */ @@ -412,6 +439,7 @@ execProg(vm_t *pThis, vmprg_t *pProg) doOP(DIV); doOP(MOD); doOP(UNARY_MINUS); + doOP(FUNC_CALL); default: ABORT_FINALIZE(RS_RET_INVALID_VMOP); dbgoprint((obj_t*) pThis, "invalid instruction %d in vmprg\n", pCurrOp->opcode); diff --git a/runtime/vmop.c b/runtime/vmop.c index fcacb15b..705acdf2 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -217,6 +217,9 @@ vmopOpcode2Str(vmop_t *pThis, uchar **ppName) case opcode_STRADD: *ppName = (uchar*) "STRADD"; break; + case opcode_FUNC_CALL: + *ppName = (uchar*) "FUNC_CALL"; + break; default: *ppName = (uchar*) "INVALID opcode"; break; diff --git a/runtime/vmop.h b/runtime/vmop.h index c3d5d5f4..938b08fd 100644 --- a/runtime/vmop.h +++ b/runtime/vmop.h @@ -59,17 +59,44 @@ typedef enum { /* do NOT start at 0 to detect uninitialized types after calloc( opcode_PUSHMSGVAR = 1002, /* requires var operand */ opcode_PUSHCONSTANT = 1003, /* requires var operand */ opcode_UNARY_MINUS = 1010, - opcode_END_PROG = 1011 + opcode_FUNC_CALL = 1012, + opcode_END_PROG = 2000 } opcode_t; +/* Additional doc, operation specific + + FUNC_CALL + All parameter passing is via the stack. Parameters are placed onto the stack in reverse order, + that means the last parameter is on top of the stack, the first at the bottom location. + At the actual top of the stack is the number of parameters. This permits functions to be + called with variable number of arguments. The function itself is responsible for poping + the right number of parameters of the stack and complaining if the number is incorrect. + On exit, a single return value must be pushed onto the stack. The FUNC_CALL operation + is generic. Its pVar argument contains the function name string (TODO: very slow, make + faster in later releases). + + Sample Function call: sampleFunc(p1, p2, p3) ; returns number 4711 (sample) + Stacklayout on entry (order is top to bottom): + 3 + p3 + p2 + p1 + ... other vars ... + + Stack on exit + 4711 + ... other vars ... + + */ + + /* the vmop object */ typedef struct vmop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ opcode_t opcode; union { - var_t *pVar; - /* TODO: add function pointer */ + var_t *pVar; /* for function call, this is the name (string) of function to be called */ } operand; struct vmop_s *pNext; /* next operation or NULL, if end of program (logically this belongs to vmprg) */ } vmop_t; -- cgit From 59192611db992e7357337beb8e68ec6cee5b3fec Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 10 Mar 2009 22:36:40 +0100 Subject: bugfix: parser did not correctly parse fields in UDP-received messages --- ChangeLog | 3 ++- plugins/imudp/imudp.c | 2 +- tools/syslogd.c | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index f1bc354c..6cab7993 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- -Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? +Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 +- bugfix: parser did not correctly parse fields in UDP-received messages - added ERE support in filter conditions new comparison operation "ereregex" - added new config directive $RepeatedMsgContainsOriginalMsg so that the diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 72450513..c7e8c1d4 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -181,7 +181,6 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* check if we have a different sender than before, if so, we need to query some new values */ if(memcmp(&frominet, frominetPrev, socklen) != 0) { CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); -DBGPRINTF("returned: fromHost '%s', FQDN: '%s'\n", fromHost, fromHostFQDN); memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ /* Here we check if a host is permitted to send us * syslog messages. If it isn't, we do not further @@ -223,6 +222,7 @@ DBGPRINTF("returned: fromHost '%s', FQDN: '%s'\n", fromHost, fromHostFQDN); MsgSetInputName(pMsg, "imudp"); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, (char*)fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); CHKiRet(submitMsg(pMsg)); diff --git a/tools/syslogd.c b/tools/syslogd.c index 6b8ce82f..eb496521 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1335,6 +1335,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) char *pBuf; int bContParse = 1; + BEGINfunc assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); p2parse = (char*) pMsg->pszUxTradMsg; @@ -1408,6 +1409,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) MsgSetMSG(pMsg, p2parse); free(pBuf); + ENDfunc return 0; /* all ok */ } -- cgit From 837b2b8d262d60580a473743a0a7955ceb980b77 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 15:06:18 +0100 Subject: preparing for 4.1.5 release --- configure.ac | 2 +- doc/manual.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 0c924754..9ba24085 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.4],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.5],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index bd927d78..064e89af 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.1.4 (devel branch) of rsyslog. +

      This documentation is for version 4.1.5 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit From dd19c937c1bfbe16063c9d633a79810944ac7eba Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 15:12:33 +0100 Subject: updated project status, version --- ChangeLog | 2 ++ doc/status.html | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6cab7993..fb0b0380 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ --------------------------------------------------------------------------- +Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +--------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages - added ERE support in filter conditions diff --git a/doc/status.html b/doc/status.html index f9e5852c..59fd0809 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,9 +5,9 @@

      This page reflects the status as of 2009-02-02.

      Current Releases

      -

      development: 4.1.4 [2009-01-29] - -change log - -download +

      development: 4.1.5 [2009-03-11] - +change log - +download
      beta: 3.21.10 [2009-02-02] - change log - -- cgit From a03d90abe1d3ac67bc5492db09e2c6e207288469 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 15:26:51 +0100 Subject: try to make file writer restartable (experimental, untested) --- tools/omfile.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/omfile.c b/tools/omfile.c index 1539ae19..4bef5dbf 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -621,20 +621,21 @@ again: */ if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) #ifdef linux - && e == EIO) { + && e == EIO #else - && e == EBADF) { + && e == EBADF #endif + ) { pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY); if (pData->fd < 0) { - iRet = RS_RET_DISABLE_ACTION; + iRet = RS_RET_SUSPENDED; errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } else { untty(); goto again; } } else { - iRet = RS_RET_DISABLE_ACTION; + iRet = RS_RET_SUSPENDED; errno = e; errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } -- cgit From f289422585ef36c39c5bc22ea20344d0c76b29ab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 19:54:30 +0100 Subject: changed one more status setting user feedback indicates it now looks like it is working ;) still some more work needed for a "good" solution --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/omfile.c b/tools/omfile.c index 4bef5dbf..28bdcf2e 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -562,7 +562,7 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa */ if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_SUSPENDED); // TODO: different state? conditional based on what went wrong? 2009-03-11 } else if(pData->fd == -1) { prepareFile(pData, pData->f_fname); } -- cgit From cdc8cf8d55f519066445ce3f9e899d092824a98e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 12 Mar 2009 05:40:03 +0100 Subject: better readability for compiled VMPrg output --- runtime/vmop.c | 51 ++++++++++++++++++++++++++++----------------------- runtime/vmprg.c | 2 +- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/runtime/vmop.c b/runtime/vmop.c index 705acdf2..441dae6c 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -73,12 +73,17 @@ ENDobjDestruct(vmop) /* DebugPrint support for the vmop object */ BEGINobjDebugPrint(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ uchar *pOpcodeName; + cstr_t *pStrVar; CODESTARTobjDebugPrint(vmop) vmopOpcode2Str(pThis, &pOpcodeName); - dbgoprint((obj_t*) pThis, "opcode: %d\t(%s), next %p, var in next line\n", (int) pThis->opcode, pOpcodeName, - pThis->pNext); - if(pThis->operand.pVar != NULL) - var.DebugPrint(pThis->operand.pVar); + CHKiRet(rsCStrConstruct(&pStrVar)); + CHKiRet(rsCStrFinish(&pStrVar)); + if(pThis->operand.pVar != NULL) { + CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + } + dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); + rsCStrDestruct(&pStrVar); +finalize_it: ENDobjDebugPrint(vmop) @@ -158,37 +163,37 @@ vmopOpcode2Str(vmop_t *pThis, uchar **ppName) *ppName = (uchar*) "and"; break; case opcode_PLUS: - *ppName = (uchar*) "+"; + *ppName = (uchar*) "add"; break; case opcode_MINUS: - *ppName = (uchar*) "-"; + *ppName = (uchar*) "sub"; break; case opcode_TIMES: - *ppName = (uchar*) "*"; + *ppName = (uchar*) "mul"; break; case opcode_DIV: - *ppName = (uchar*) "/"; + *ppName = (uchar*) "div"; break; case opcode_MOD: - *ppName = (uchar*) "%"; + *ppName = (uchar*) "mod"; break; case opcode_NOT: *ppName = (uchar*) "not"; break; case opcode_CMP_EQ: - *ppName = (uchar*) "=="; + *ppName = (uchar*) "cmp_=="; break; case opcode_CMP_NEQ: - *ppName = (uchar*) "!="; + *ppName = (uchar*) "cmp_!="; break; case opcode_CMP_LT: - *ppName = (uchar*) "<"; + *ppName = (uchar*) "cmp_<"; break; case opcode_CMP_GT: - *ppName = (uchar*) ">"; + *ppName = (uchar*) "cmp_>"; break; case opcode_CMP_LTEQ: - *ppName = (uchar*) "<="; + *ppName = (uchar*) "cmp_<="; break; case opcode_CMP_CONTAINS: *ppName = (uchar*) "contains"; @@ -197,31 +202,31 @@ vmopOpcode2Str(vmop_t *pThis, uchar **ppName) *ppName = (uchar*) "startswith"; break; case opcode_CMP_GTEQ: - *ppName = (uchar*) ">="; + *ppName = (uchar*) "cmp_>="; break; case opcode_PUSHSYSVAR: - *ppName = (uchar*) "PUSHSYSVAR"; + *ppName = (uchar*) "push_sysvar"; break; case opcode_PUSHMSGVAR: - *ppName = (uchar*) "PUSHMSGVAR"; + *ppName = (uchar*) "push_msgvar"; break; case opcode_PUSHCONSTANT: - *ppName = (uchar*) "PUSHCONSTANT"; + *ppName = (uchar*) "push_const"; break; case opcode_POP: - *ppName = (uchar*) "POP"; + *ppName = (uchar*) "pop"; break; case opcode_UNARY_MINUS: - *ppName = (uchar*) "UNARY_MINUS"; + *ppName = (uchar*) "unary_minus"; break; case opcode_STRADD: - *ppName = (uchar*) "STRADD"; + *ppName = (uchar*) "strconcat"; break; case opcode_FUNC_CALL: - *ppName = (uchar*) "FUNC_CALL"; + *ppName = (uchar*) "func_call"; break; default: - *ppName = (uchar*) "INVALID opcode"; + *ppName = (uchar*) "!invalid_opcode!"; break; } diff --git a/runtime/vmprg.c b/runtime/vmprg.c index 705e6948..75915025 100644 --- a/runtime/vmprg.c +++ b/runtime/vmprg.c @@ -74,7 +74,7 @@ ENDobjDestruct(vmprg) BEGINobjDebugPrint(vmprg) /* be sure to specify the object type also in END and CODESTART macros! */ vmop_t *pOp; CODESTARTobjDebugPrint(vmprg) - dbgoprint((obj_t*) pThis, "program contents:\n"); + dbgoprint((obj_t*) pThis, "VM Program:\n"); for(pOp = pThis->vmopRoot ; pOp != NULL ; pOp = pOp->pNext) { vmop.DebugPrint(pOp); } -- cgit From 2bb202f665df594286595e226251b3580b474a4d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 13 Mar 2009 12:52:53 +0100 Subject: bugfix: removed (newly introduced) memory leaks --- runtime/vm.c | 1 + runtime/vmop.c | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/runtime/vm.c b/runtime/vm.c index 113a9d59..a25476c2 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -354,6 +354,7 @@ RUNLOG_VAR("%d", iStrlen); } else ABORT_FINALIZE(RS_RET_INVLD_FUNC); PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME + var.Destruct(&numOperands); /* no longer needed */ finalize_it: ENDop(FUNC_CALL) diff --git a/runtime/vmop.c b/runtime/vmop.c index 441dae6c..a343481e 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -61,12 +61,8 @@ rsRetVal vmopConstructFinalize(vmop_t __attribute__((unused)) *pThis) /* destructor for the vmop object */ BEGINobjDestruct(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(vmop) - if( pThis->opcode == opcode_PUSHSYSVAR - || pThis->opcode == opcode_PUSHMSGVAR - || pThis->opcode == opcode_PUSHCONSTANT) { - if(pThis->operand.pVar != NULL) - var.Destruct(&pThis->operand.pVar); - } + if(pThis->operand.pVar != NULL) + var.Destruct(&pThis->operand.pVar); ENDobjDestruct(vmop) -- cgit From 446a982c2b8f77eb0e349e5bd8240ece6a0772bd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Mar 2009 16:05:15 +0100 Subject: some more information on rainerscript imlementation (taken from old rscript branch, which is now obsolete) --- doc/rscript_abnf.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/rscript_abnf.html b/doc/rscript_abnf.html index 278fb59c..d60edb5c 100644 --- a/doc/rscript_abnf.html +++ b/doc/rscript_abnf.html @@ -30,6 +30,8 @@ table... values('&$facility&','&$severity&...?]
        endact table... values('&$facility&','&$severity&...?]
        )

      ... or ...

      define action writeMySQL(
          type='ommysql.so', queue.mode='disk', queue.highwatermark = 300,
          action.dbname='events', action.dbuser='uid',
          [?action.template='templatename'?] or [?action.sql='insert into table... values('&$facility&','&$severity&...?]
         )

      if $message contains "error" then action writeMySQL(action.dbname='differentDB')

      [rsyslog.conf overview] +

      Implementation

      +RainerScript will be implemented via a hand-crafted LL(1) parser. I was tempted to use yacc, but it turned out the resulting code was not thread-safe and as such did not fit within the context of rsyslog. Also, limited error handling is not a real problem for us: if there is a problem in parsing the configuration file, we stop processing. Guessing what was meant and trying to recover would IMHO not be good choices for something like a syslogd. [manual index] [rsyslog site]

      This documentation is part of the rsyslog -- cgit From 208f4e107c78b00c3bdf09301d1b1e43e1b9fdf8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Mar 2009 17:26:45 +0100 Subject: added output module template so far, this is mostly some documentation, but I hope that during the process of creating output modules out of it we will get good questions and thus can extend the template. In any case, it should be better than what we had so far... --- ChangeLog | 5 +- Makefile.am | 6 +- configure.ac | 29 +++++- plugins/omtemplate/Makefile.am | 8 ++ plugins/omtemplate/omtemplate.c | 220 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 plugins/omtemplate/Makefile.am create mode 100644 plugins/omtemplate/omtemplate.c diff --git a/ChangeLog b/ChangeLog index 3488dfb2..8c8442e2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +--------------------------------------------------------------------------- +Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet @@ -5,8 +7,7 @@ NOTE: nested function calls are not yet supported due to a design issue with the function call VM instruction set design. - implemented the strlen() RainerScript function ---------------------------------------------------------------------------- -Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +- added a template output module --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/Makefile.am b/Makefile.am index aaca570d..7bb6af8e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -88,6 +88,10 @@ if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif +if ENABLE_OMTEMPLATE +SUBDIRS += plugins/omtemplate +endif + if ENABLE_IMFILE SUBDIRS += plugins/imfile endif @@ -114,5 +118,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 4763fcd5..0c9483ca 100644 --- a/configure.ac +++ b/configure.ac @@ -631,9 +631,7 @@ AC_SUBST(LIBLOGGING_CFLAGS) AC_SUBST(LIBLOGGING_LIBS) -# settings for the template input module; copy and modify this code -# if you intend to add your own module. Be sure to replace imtemplate -# by the actual name of your module. +# settings for the file input module; AC_ARG_ENABLE(imfile, [AS_HELP_STRING([--enable-imfile],[file input module enabled @<:@default=no@:>@])], [case "${enableval}" in @@ -649,8 +647,7 @@ AC_ARG_ENABLE(imfile, # AM_CONDITIONAL(ENABLE_IMFILE, test x$enable_imfile = xyes) -AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) -# end of copy template - be sure to serach for imtemplate to find everything! + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -671,6 +668,26 @@ AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to serach for imtemplate to find everything! +# settings for the template output module; copy and modify this code +# if you intend to add your own module. Be sure to replace omtemplate +# by the actual name of your module. +AC_ARG_ENABLE(omtemplate, + [AS_HELP_STRING([--enable-omtemplate],[Compiles omtemplate template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omtemplate="yes" ;; + no) enable_omtemplate="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omtemplate) ;; + esac], + [enable_omtemplate=no] +) +# +# you may want to do some library checks here - see snmp, mysql, pgsql modules +# for samples +# +AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) +# end of copy template - be sure to serach for omtemplate to find everything! + + AC_CONFIG_FILES([Makefile \ runtime/Makefile \ tools/Makefile \ @@ -683,6 +700,7 @@ AC_CONFIG_FILES([Makefile \ plugins/immark/Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ + plugins/omtemplate/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ @@ -713,6 +731,7 @@ echo "RELP support enabled: $enable_relp" echo "imdiag enabled: $enable_imdiag" echo "file input module enabled: $enable_imfile" echo "input template module will be compiled: $enable_imtemplate" +echo "output template module will be compiled: $enable_omtemplate" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" echo "GnuTLS network stream driver enabled: $enable_gnutls" diff --git a/plugins/omtemplate/Makefile.am b/plugins/omtemplate/Makefile.am new file mode 100644 index 00000000..58ddc794 --- /dev/null +++ b/plugins/omtemplate/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omtemplate.la + +omtemplate_la_SOURCES = omtemplate.c omtemplate.h +omtemplate_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omtemplate_la_LDFLAGS = -module -avoid-version +omtemplate_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omtemplate/omtemplate.c b/plugins/omtemplate/omtemplate.c new file mode 100644 index 00000000..e35968ad --- /dev/null +++ b/plugins/omtemplate/omtemplate.c @@ -0,0 +1,220 @@ +/* omtemplate.c + * This is a template for an output module. It implements a very + * simple single-threaded output, just as thought of by the output + * plugin interface. + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-03-16 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +typedef struct _instanceData { + /* here you need to define all action-specific data. A record of type + * instanceData will be handed over to each instance of the action. Keep + * in mind that there may be several invocations of the same type of action + * inside rsyslog.conf, and this is what keeps them apart. Do NOT use + * static data for this! + */ + unsigned iSrvPort; /* sample: server port */ +} instanceData; + +/* config variables + * For the configuration interface, we need to keep track of some settings. This + * is done in global variables. It works as follows: when configuration statements + * are entered, the config file handler (or custom function) sets the global + * variable here. When the action then actually is instantiated, this handler + * copies over to instanceData whatever configuration settings (from the global + * variables) apply. The global variables are NEVER used inside an action + * instance (at least this is how it is supposed to work ;) + */ +static int iSrvPort = 0; /* sample: server port */ + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + /* use this to specify if select features are supported by this + * plugin. If not, the framework will handle that. Currently, only + * RepeatedMsgReduction ("last message repeated n times") is optional. + */ + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance + /* this is a cleanup callback. All dynamically-allocated resources + * in instance data must be cleaned up here. Prime examples are + * malloc()ed memory, file & database handles and the like. + */ +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + /* permits to spit out some debug info */ +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume + /* this is called when an action has been suspended and the + * rsyslog core tries to resume it. The action must then + * retry (if possible) and report RS_RET_OK if it succeeded + * or RS_RET_SUSPENDED otherwise. + * Note that no data can be written in this callback, as it is + * not present. Prime examples of what can be retried are + * reconnects to remote hosts, reconnects to database, + * opening of files and the like. + * If there is no retry-type of operation, the action may + * return RS_RET_OK, so that it will get called on its doAction + * entry point (where it receives data), retries there, and + * immediately returns RS_RET_SUSPENDED if that does not work + * out. This disables some optimizations in the core's retry logic, + * but is a valid and expected behaviour. Note that it is also OK + * for the retry entry point to return OK but the immediately following + * doAction call to fail. In real life, for example, a buggy com line + * may cause such behaviour. + * Note that there is no guarantee that the core will very quickly + * call doAction after the retry succeeded. Today, it does, but that may + * not always be the case. + */ +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + /* this is where you receive the message and need to carry out the + * action. Data is provided in ppString[i] where 0 <= i <= num of strings + * requested. + * Return RS_RET_OK if all goes well, RS_RET_SUSPENDED if the action can + * currently not complete, or an error code or RS_RET_DISABLED. The later + * two should only be returned if there is no hope that the action can be + * restored unless an rsyslog restart (prime example is an invalid config). + * Error code or RS_RET_DISABLED permanently disables the action, up to + * the next restart. + */ +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us + * This is a clumpsy interface. We receive the action-part of the selector line + * and need to look at the first characters. If they match our signature + * ":omtemplate:", then we need to instantiate an action. It is recommended that + * newer actions just watch for the template and all other parameters are passed in + * via $-config-lines, this will hopefully be compatbile with future config syntaxes. + * If we do not detect our signature, we must return with RS_RET_CONFLINE_UNPROCESSED + * and NOT do anything else. + */ + if(strncmp((char*) p, ":omtemplate:", sizeof(":omtemplate:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omtemplate:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + /* if we have, call rsyslog runtime to get us template. Note that StdFmt below is + * the standard name. Currently, we may need to patch tools/syslogd.c if we need + * to add a new standard template. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, (uchar*) " StdFmt")); + + /* if we reach this point, all went well, and we can copy over to instanceData + * those configuration elements that we need. + */ + pData->iSrvPort = (unsigned) iSrvPort; /* set configured port */ + +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +/* Reset config variables for this module to default values. + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + iSrvPort = 0; /* zero is the default port */ + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + /* register our config handlers */ + /* confguration parameters MUST always be specified in lower case! */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomtemplteserverport", 0, eCmdHdlrInt, NULL, &iSrvPort, STD_LOADABLE_MODULE_ID)); + /* "resetconfigvariables" should be provided. Notat that it is a chained directive */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ -- cgit From 3762d2b7bc669e45d51a2157158e3cc925dd0152 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 Mar 2009 18:26:15 +0100 Subject: fixed some message-loss situations when file write failures happened --- rsyslog.conf | 1 + tools/omfile.c | 12 +++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/rsyslog.conf b/rsyslog.conf index 47fc4402..329704b8 100644 --- a/rsyslog.conf +++ b/rsyslog.conf @@ -58,3 +58,4 @@ local7.* /var/log/boot.log # UDP Syslog Server: #$ModLoad imudp.so # provides UDP syslog reception #$UDPServerRun 514 # start a UDP syslog server at standard port 514 +*.* /mnt/logfile diff --git a/tools/omfile.c b/tools/omfile.c index 28bdcf2e..8be815f2 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -600,20 +600,14 @@ again: } } - if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { + if(write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { int e = errno; +dbgprintf("++++++++++ log file writer error %d\n", e); /* If a named pipe is full, just ignore it for now - mrn 24 May 96 */ if (pData->fileType == eTypePIPE && e == EAGAIN) - ABORT_FINALIZE(RS_RET_OK); - - /* If the filesystem is filled up, just ignore - * it for now and continue writing when possible - * based on patch for sysklogd by Martin Schulze on 2007-05-24 - */ - if (pData->fileType == eTypeFILE && e == ENOSPC) - ABORT_FINALIZE(RS_RET_OK); + ABORT_FINALIZE(RS_RET_SUSPENDED); (void) close(pData->fd); /* Check for EBADF on TTY's due to vhangup() -- cgit From 7268b9bc5b8a7d4ea58cbec7fb6180574e2b20d6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 Mar 2009 18:27:33 +0100 Subject: undid typo --- rsyslog.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/rsyslog.conf b/rsyslog.conf index 329704b8..47fc4402 100644 --- a/rsyslog.conf +++ b/rsyslog.conf @@ -58,4 +58,3 @@ local7.* /var/log/boot.log # UDP Syslog Server: #$ModLoad imudp.so # provides UDP syslog reception #$UDPServerRun 514 # start a UDP syslog server at standard port 514 -*.* /mnt/logfile -- cgit From 27a639ed8875969574543f7738c7bcb14c6149d2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 18 Mar 2009 17:55:15 +0100 Subject: omfile bugfixing - fixed a bug that caused action retries not to work correctly situation was only cleared by a restart - bugfix: closed dynafile was potentially never written until another dynafile name was generated - potential loss of messages --- ChangeLog | 4 ++++ action.c | 8 +++++++- doc/rsyslog_conf_global.html | 1 + tools/omfile.c | 20 ++++++++++++++++---- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 18bb0e53..ae700656 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +- fixed a bug that caused action retries not to work correctly + situation was only cleared by a restart +- bugfix: closed dynafile was potentially never written until another + dynafile name was generated - potential loss of messages --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/action.c b/action.c index a41f976c..10607e72 100644 --- a/action.c +++ b/action.c @@ -353,7 +353,13 @@ static rsRetVal actionTryResume(action_t *pThis) ASSERT(pThis != NULL); - ttNow = getActNow(pThis); /* cache "now" */ + /* for resume handling, we must always obtain a fresh timestamp. We used + * to use the action timestamp, but in this case we will never reach a + * point where a resumption is actually tried, because the action timestamp + * is always in the past. So we can not avoid doing a fresh time() call + * here. -- rgerhards, 2009-03-18 + */ + time(&ttNow); /* cache "now" */ /* first check if it is time for a re-try */ if(ttNow > pThis->ttResumeRtry) { diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index b0c1e400..d011bd2b 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -103,6 +103,7 @@ default 60000 (1 minute)]

    • $DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
    • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
    • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
      • +
    • $CreateDirs [on/off] - create directories on an as-needed basis
    • $DirCreateMode
    • $DirGroup
    • $DirOwner
      • diff --git a/tools/omfile.c b/tools/omfile.c index 8be815f2..bf84d1a7 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -457,6 +457,8 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg int i; int iFirstFree; dynaFileCacheEntry **pCache; + + BEGINfunc ASSERT(pData != NULL); ASSERT(newFileName != NULL); @@ -542,6 +544,8 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); + ENDfunc + return 0; } @@ -553,6 +557,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) { off_t actualFileSize; + int iLenWritten; DEFiRet; ASSERT(pData != NULL); @@ -563,7 +568,9 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) ABORT_FINALIZE(RS_RET_SUSPENDED); // TODO: different state? conditional based on what went wrong? 2009-03-11 - } else if(pData->fd == -1) { + } + + if(pData->fd == -1) { prepareFile(pData, pData->f_fname); } @@ -600,16 +607,21 @@ again: } } - if(write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { + iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); +//dbgprintf("lenwritten: %d\n", iLenWritten); + if(iLenWritten < 0) { int e = errno; -dbgprintf("++++++++++ log file writer error %d\n", e); + char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); /* If a named pipe is full, just ignore it for now - mrn 24 May 96 */ - if (pData->fileType == eTypePIPE && e == EAGAIN) + if(pData->fileType == eTypePIPE && e == EAGAIN) ABORT_FINALIZE(RS_RET_SUSPENDED); (void) close(pData->fd); + pData->fd = -1; /* tell that fd is no longer open! */ /* Check for EBADF on TTY's due to vhangup() * Linux uses EIO instead (mrn 12 May 96) */ -- cgit From 508a9e0cef064b082cd9e13aecd9d6c0f1f51977 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 12:01:43 +0100 Subject: omfile suspend handling for non-dynafiles, also bugfixes primarily bugs introduced by recent changes. We now also handle static file names correctly, that was not the case before. We now correctly reset the descriptor in the dynafile cache if somthing goes wrong. Keep in mind that reliablity of output is depending on the reliability of the file system driver (the cifs driver returns OK, but still loses data if it is disconnected for too-long). --- ChangeLog | 3 +++ tools/omfile.c | 44 ++++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index ae700656..5183674f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,9 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? situation was only cleared by a restart - bugfix: closed dynafile was potentially never written until another dynafile name was generated - potential loss of messages +- improved omfile so that it properly suspends itself if there is an + i/o or file name generation error. This enables it to be used with + the full high availability features of rsyslog's engine --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/tools/omfile.c b/tools/omfile.c index bf84d1a7..ff4c4618 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -383,9 +383,12 @@ static void dynaFileFreeCache(instanceData *pData) * file access, which, among others, means the the file wil be opened * and any directories in between will be created (based on config, of * course). -- rgerhards, 2008-10-22 + * changed to iRet interface - 2009-03-19 */ -static void prepareFile(instanceData *pData, uchar *newFileName) +static rsRetVal +prepareFile(instanceData *pData, uchar *newFileName) { + DEFiRet; if(pData->fileType == eTypePIPE) { pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); FINALIZE; /* we are done in this case */ @@ -399,14 +402,14 @@ static void prepareFile(instanceData *pData, uchar *newFileName) pData->fd = -1; /* file does not exist, create it (and eventually parent directories */ if(pData->bCreateDirs) { - /* we fist need to create parent dirs if they are missing + /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code * fall through to error handler below. */ if(makeFileParentDirs(newFileName, strlen((char*)newFileName), pData->fDirCreateMode, pData->dirUID, pData->dirGID, pData->bFailOnChown) != 0) { - return; /* we give up */ + ABORT_FINALIZE(RS_RET_ERR); /* we give up */ } } /* no matter if we needed to create directories or not, we now try to create @@ -418,8 +421,7 @@ static void prepareFile(instanceData *pData, uchar *newFileName) /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, - pData->fileGID) != 0) { + if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; close(pData->fd); @@ -434,11 +436,17 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } } finalize_it: - if((pData->fd) != 0 && isatty(pData->fd)) { + /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean + * non-open file descriptor. Anyhow, I leave this comment for the time being to that if + * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 + */ + if(pData->fd != -1 && isatty(pData->fd)) { DBGPRINTF("file %d is a tty file\n", pData->fd); pData->fileType = eTypeTTY; untty(); } + + RETiRet; } @@ -520,7 +528,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } /* Ok, we finally can open the file */ - prepareFile(pData, newFileName); + prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ /* file is either open now or an error state set */ if(pData->fd == -1) { @@ -567,11 +575,14 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa */ if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); // TODO: different state? conditional based on what went wrong? 2009-03-11 + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } if(pData->fd == -1) { - prepareFile(pData, pData->f_fname); + rsRetVal iRetLocal; + iRetLocal = prepareFile(pData, pData->f_fname); + if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } /* create the message based on format specified */ @@ -615,13 +626,19 @@ again: rs_strerror_r(errno, errStr, sizeof(errStr)); DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); - /* If a named pipe is full, just ignore it for now - - mrn 24 May 96 */ + /* If a named pipe is full, we suspend this action for a while */ if(pData->fileType == eTypePIPE && e == EAGAIN) ABORT_FINALIZE(RS_RET_SUSPENDED); - (void) close(pData->fd); + close(pData->fd); pData->fd = -1; /* tell that fd is no longer open! */ + if(pData->bDynamicName && pData->iCurrElt != -1) { + /* in this case, we need to invalidate the name in the cache, too + * otherwise, an invalid fd may show up if we had a file name change. + * rgerhards, 2009-03-19 + */ + pData->dynCache[pData->iCurrElt]->fd = -1; + } /* Check for EBADF on TTY's due to vhangup() * Linux uses EIO instead (mrn 12 May 96) */ @@ -793,6 +810,9 @@ CODESTARTparseSelectorAct pData->dirUID = dirUID; pData->dirGID = dirGID; + /* at this stage, we ignore the return value of prepareFile, this is taken + * care of in later steps. -- rgerhards, 2009-03-19 + */ prepareFile(pData, pData->f_fname); if(pData->fd < 0 ) { -- cgit From 935018ed625bc4cf1d6b28fa16e0986078029aaf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 13:58:39 +0100 Subject: adapted test framework to new script engine --- Makefile.am | 2 ++ tests/1.rstest | 34 +++++++++++++++++----------------- tests/2.rstest | 4 ++-- tests/3.rstest | 21 +++++++++++++++++++++ tests/Makefile.am | 2 +- tests/rscript.c | 5 +++-- 6 files changed, 46 insertions(+), 22 deletions(-) create mode 100644 tests/3.rstest diff --git a/Makefile.am b/Makefile.am index 7bb6af8e..87e378ee 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,3 +1,5 @@ +AUTOMAKE_OPTIONS=dejagnu + sbin_PROGRAMS = pkglib_LTLIBRARIES = diff --git a/tests/1.rstest b/tests/1.rstest index 5c152589..4716e8b3 100644 --- a/tests/1.rstest +++ b/tests/1.rstest @@ -4,23 +4,23 @@ in: 'test 1' <> $var or /* some comment */($SEVERITY == -4 +5 -(3 * - 2) and $fromhost == '127.0.0.1') then $$$ out: -00000000: PUSHCONSTANT test 1[cstr] -00000001: PUSHMSGVAR var[cstr] -00000002: != -00000003: PUSHMSGVAR severity[cstr] -00000004: PUSHCONSTANT 4[nbr] -00000005: UNARY_MINUS -00000006: PUSHCONSTANT 5[nbr] -00000007: + -00000008: PUSHCONSTANT 3[nbr] -00000009: PUSHCONSTANT 2[nbr] -00000010: UNARY_MINUS -00000011: * -00000012: - -00000013: == -00000014: PUSHMSGVAR fromhost[cstr] -00000015: PUSHCONSTANT 127.0.0.1[cstr] -00000016: == +00000000: push_const test 1[cstr] +00000001: push_msgvar var[cstr] +00000002: cmp_!= +00000003: push_msgvar severity[cstr] +00000004: push_const 4[nbr] +00000005: unary_minus +00000006: push_const 5[nbr] +00000007: add +00000008: push_const 3[nbr] +00000009: push_const 2[nbr] +00000010: unary_minus +00000011: mul +00000012: sub +00000013: cmp_== +00000014: push_msgvar fromhost[cstr] +00000015: push_const 127.0.0.1[cstr] +00000016: cmp_== 00000017: and 00000018: or $$$ diff --git a/tests/2.rstest b/tests/2.rstest index 7fb5b799..f0e8205b 100644 --- a/tests/2.rstest +++ b/tests/2.rstest @@ -4,7 +4,7 @@ in: $msg contains 'test' then $$$ out: -00000000: PUSHMSGVAR msg[cstr] -00000001: PUSHCONSTANT test[cstr] +00000000: push_msgvar msg[cstr] +00000001: push_const test[cstr] 00000002: contains $$$ diff --git a/tests/3.rstest b/tests/3.rstest new file mode 100644 index 00000000..93cb941a --- /dev/null +++ b/tests/3.rstest @@ -0,0 +1,21 @@ +# a simple RainerScript test +result: 0 +in: +strlen($msg & strlen('abc')) > 20 +30 + -40 then +$$$ +out: +00000000: push_msgvar msg[cstr] +00000001: push_const abc[cstr] +00000002: push_const 1[nbr] +00000003: func_call strlen[cstr] +00000004: strconcat +00000005: push_const 1[nbr] +00000006: func_call strlen[cstr] +00000007: push_const 20[nbr] +00000008: push_const 30[nbr] +00000009: add +00000010: push_const 40[nbr] +00000011: unary_minus +00000012: add +00000013: cmp_> +$$$ diff --git a/tests/Makefile.am b/tests/Makefile.am index 14e7c195..7a31be45 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -4,7 +4,7 @@ TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ #TESTS = $(check_PROGRAMS) test_files = testbench.h runtime-dummy.c -EXTRA_DIST=1.rstest 2.rstest err1.rstest \ +EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ cfg.sh \ cfg1.cfgtest \ cfg1.testin \ diff --git a/tests/rscript.c b/tests/rscript.c index d4e8caeb..3eec9c3c 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -101,9 +101,10 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) CHKiRet(vmprg.Obj2Str(pExpr->pVmprg, pstrPrg)); if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { + int iLen; printf("error: compiled program different from expected result!\n"); - printf("generated vmprg:\n%s\n", rsCStrGetSzStr(pstrPrg)); - printf("expected:\n%s\n", rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } -- cgit From 790532bb834e3d4fc07273b2d72127e39ef3e142 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 14:02:00 +0100 Subject: fixed broken make distcheck due to invalidly stated omtemplate file --- plugins/omtemplate/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/omtemplate/Makefile.am b/plugins/omtemplate/Makefile.am index 58ddc794..e816c7c6 100644 --- a/plugins/omtemplate/Makefile.am +++ b/plugins/omtemplate/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = omtemplate.la -omtemplate_la_SOURCES = omtemplate.c omtemplate.h +omtemplate_la_SOURCES = omtemplate.c omtemplate_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) omtemplate_la_LDFLAGS = -module -avoid-version omtemplate_la_LIBADD = -- cgit From bbfa04fbe63f1bbb47f5cdc683045cf2775b3977 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 17:50:07 +0100 Subject: improved testing support worked on ways to provide a better test suite: - added -T rsyslogd command line option, enables to specify a directory where to chroot() into on startup. This is NOT a security feature but introduced to support testing. Thus, -T does not make sure chroot() is used in a secure way. (may be removed later) - added omstdout module for testing purposes. Spits out all messages to stdout - no config option, no other features - modified $ModLoad statement so that for modules whom's name starts with a dot, no path is prepended (this enables relative-pathes and should not break any valid current config) --- ChangeLog | 9 ++++ Makefile.am | 8 +-- configure.ac | 21 +++++++- plugins/omstdout/Makefile.am | 8 +++ plugins/omstdout/omstdout.c | 125 +++++++++++++++++++++++++++++++++++++++++++ runtime/modules.c | 2 +- tests/Makefile.am | 3 ++ tools/syslogd.c | 17 +++++- 8 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 plugins/omstdout/Makefile.am create mode 100644 plugins/omstdout/omstdout.c diff --git a/ChangeLog b/ChangeLog index 8c8442e2..1cfb3c3a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,15 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? issue with the function call VM instruction set design. - implemented the strlen() RainerScript function - added a template output module +- added -T rsyslogd command line option, enables to specify a directory + where to chroot() into on startup. This is NOT a security feature but + introduced to support testing. Thus, -T does not make sure chroot() + is used in a secure way. (may be removed later) +- added omstdout module for testing purposes. Spits out all messages to + stdout - no config option, no other features +- modified $ModLoad statement so that for modules whom's name starts with + a dot, no path is prepended (this enables relative-pathes and should + not break any valid current config) --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/Makefile.am b/Makefile.am index 87e378ee..97f4aed3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,3 @@ -AUTOMAKE_OPTIONS=dejagnu - sbin_PROGRAMS = pkglib_LTLIBRARIES = @@ -90,6 +88,10 @@ if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif +if ENABLE_OMSTDOUT +SUBDIRS += plugins/omstdout +endif + if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif @@ -120,5 +122,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 0c9483ca..fcd935e8 100644 --- a/configure.ac +++ b/configure.ac @@ -688,6 +688,23 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to serach for omtemplate to find everything! +# settings for omstdout +AC_ARG_ENABLE(omstdout, + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omstdout="yes" ;; + no) enable_omstdout="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omstdout) ;; + esac], + [enable_omstdout=no] +) +# +# you may want to do some library checks here - see snmp, mysql, pgsql modules +# for samples +# +AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) + + AC_CONFIG_FILES([Makefile \ runtime/Makefile \ tools/Makefile \ @@ -701,6 +718,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ plugins/omtemplate/Makefile \ + plugins/omstdout/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ @@ -731,7 +749,8 @@ echo "RELP support enabled: $enable_relp" echo "imdiag enabled: $enable_imdiag" echo "file input module enabled: $enable_imfile" echo "input template module will be compiled: $enable_imtemplate" -echo "output template module will be compiled: $enable_omtemplate" +echo "output template module will be compiled: $enable_omtemplate" +echo "omstdout module will be compiled: $enable_omstdout" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" echo "GnuTLS network stream driver enabled: $enable_gnutls" diff --git a/plugins/omstdout/Makefile.am b/plugins/omstdout/Makefile.am new file mode 100644 index 00000000..9f5d497f --- /dev/null +++ b/plugins/omstdout/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omstdout.la + +omstdout_la_SOURCES = omstdout.c +omstdout_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omstdout_la_LDFLAGS = -module -avoid-version +omstdout_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c new file mode 100644 index 00000000..6e227ba9 --- /dev/null +++ b/plugins/omstdout/omstdout.c @@ -0,0 +1,125 @@ +/* omstdout.c + * send all output to stdout - this is primarily a test driver (but may + * be used for weired use cases). Not tested for robustness! + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-03-19 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + write(1, (char*)ppString[0], strlen((char*)ppString[0])); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omstdout:", sizeof(":omstdout:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omstdout:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit + +/* vi:set ai: + */ diff --git a/runtime/modules.c b/runtime/modules.c index d548a949..cef4eac6 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -608,7 +608,7 @@ Load(uchar *pModName) iLoadCnt = 0; do { /* now build our load module name */ - if(*pModName == '/') { + if(*pModName == '/' || *pModName == '.') { *szPath = '\0'; /* we do not need to append the path - its already in the module name */ iPathLen = 0; } else { diff --git a/tests/Makefile.am b/tests/Makefile.am index 7a31be45..384afd4e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,3 +1,6 @@ +#AUTOMAKE_OPTIONS=dejagnu +#DEJATOOL=Rainer + check_PROGRAMS = rt_init rscript TESTS = $(check_PROGRAMS) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ diff --git a/tools/syslogd.c b/tools/syslogd.c index 235bc52e..9f962899 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3324,7 +3324,7 @@ int realMain(int argc, char **argv) * only when actually neeeded. * rgerhards, 2008-04-04 */ - while((ch = getopt(argc, argv, "46a:Ac:def:g:hi:l:m:M:nN:op:qQr::s:t:u:vwx")) != EOF) { + while((ch = getopt(argc, argv, "46a:Ac:def:g:hi:l:m:M:nN:op:qQr::s:t:T:u:vwx")) != EOF) { switch((char)ch) { case '4': case '6': @@ -3342,6 +3342,7 @@ int realMain(int argc, char **argv) case 'q': /* add hostname if DNS resolving has failed */ case 'Q': /* dont resolve hostnames in ACL to IPs */ case 's': + case 'T': /* chroot on startup (primarily for testing) */ case 'u': /* misc user settings */ case 'w': /* disable disallowed host warnings */ case 'x': /* disable dns for remote messages */ @@ -3586,6 +3587,20 @@ int realMain(int argc, char **argv) } else fprintf(stderr, "-t option only supported in compatibility modes 0 to 2 - ignored\n"); break; + case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ +{ +char buf[1024]; +getcwd(buf, 1024); +printf("pwd: '%s'\n", buf); +printf("chroot to '%s'\n", arg); + if(chroot(arg) != 0) { + perror("chroot"); + exit(1); + } +getcwd(buf, 1024); +printf("pwd: '%s'\n", buf); +} + break; case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); if(iHelperUOpt & 0x01) -- cgit From c11b7ec7d6e6216481f28447b94e59e524dd824c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 23 Mar 2009 11:27:38 +0100 Subject: some text scripts added (experimental) --- tests/test.tcl | 41 +++++++++++++++++++++++++++++++++++++++++ tests/testruns/parser.conf | 10 ++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/test.tcl create mode 100644 tests/testruns/parser.conf diff --git a/tests/test.tcl b/tests/test.tcl new file mode 100644 index 00000000..7f1b7a6b --- /dev/null +++ b/tests/test.tcl @@ -0,0 +1,41 @@ +# rsyslog parser tests +package require Expect +package require udp 1.0 + +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; +#interact; +#puts "pid: $rsyslogdPID"; +#sleep 1; +#expect "\n"; +expect "}}"; # eat startup message +set udpSock [udp_open]; +udp_conf $udpSock 127.0.0.1 514 +set files [glob *.parse1] +puts "done init\n"; + + +foreach testcase $files { + puts "File $testcase"; + set fp [open "$testcase" r]; + fconfigure $fp -buffering line + #set data [read $fp]; + #set data [split $data "\n"]; + gets $fp input + puts "Line 1: $input\n"; + + puts $udpSock $input; + flush $udpSock; + + + set i 1 + expect -re "{{.*}}"; + set result $expect_out(buffer); + + #puts "MSG $i: '$expect_out(buffer)'"; + puts "MSG $i: '$result'\n"; + set i [expr {$i + 1}]; + +} + +exec kill $rsyslogdPID; +close $udpSock; diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf new file mode 100644 index 00000000..a515ff6c --- /dev/null +++ b/tests/testruns/parser.conf @@ -0,0 +1,10 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imuxsock/.libs/imuxsock +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 514 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" +*.* :omstdout:;expect -- cgit From de3341a64131092fabb3a1c60adb77e70bdb7fb6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 23 Mar 2009 18:24:44 +0100 Subject: added "rsyslog family tree" graph of rsyslog versions and branches --- doc/rsyslog-vers.dot | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 doc/rsyslog-vers.dot diff --git a/doc/rsyslog-vers.dot b/doc/rsyslog-vers.dot new file mode 100644 index 00000000..a5563f94 --- /dev/null +++ b/doc/rsyslog-vers.dot @@ -0,0 +1,82 @@ +// This file is part of rsyslog. +// +// rsyslog "family tree" compressed version +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot rsyslog-vers.dot -Tpng >rsyslog-vers.png + +digraph G { + label="\n\nrsyslog \"family tree\"\nhttp://www.rsyslog.com"; + fontsize=20; + + v1stable [label="v1-stable", shape=box, style=dotted]; + v2stable [label="v2-stable", shape=box, style=filled]; + v3stable [label="v3-stable", shape=box, style=filled]; + beta [label="beta", shape=box, style=filled]; + devel [label="devel", shape=box, style=filled]; + "1.0.5" [style=dotted]; + perf [style=dashed]; + "imudp-select" [style=dashed label="imudp-\nselect"]; + msgnolock [style=dashed]; + "file-errHdlr" [style=dashed]; + solaris [style=dashed]; + tests [style=dashed]; + "0.x.y" [group=master]; + "1.10.0" [group=master]; + "1.21.2" [group=master]; + "3.10.0" [group=master]; + "3.15.1" [group=master]; + "3.17.0" [group=master]; + "3.19.x" [group=master]; + "3.21.x" [group=master]; + "4.1.0" [group=master]; + "4.1.4" [group=master]; + "4.1.5" [group=master]; + "4.1.6" [group=master]; + + sysklogd -> "0.x.y" [color=red]; + "0.x.y" -> "1.0.0"; + "0.x.y" -> "1.10.0" [color=red]; + "1.0.0" -> "1.0.5" [style=dashed]; + "1.10.0" -> "1.21.2" [color=red style=dashed]; + "1.21.2" -> "2.0.0" [color=blue]; + "2.0.0" -> "2.0.5" [style=dashed, color=blue]; + "1.21.2" -> "3.10.0" [color=red]; + "3.10.0" -> "3.15.1" [color=red style=dashed]; + "3.15.1" -> "tests"; + "3.15.1" -> "3.17.0" [color=red style=dashed]; + "3.15.1" -> "3.16.x"; + "3.16.x" -> "3.18.x" [color=blue, style=dashed]; + "3.17.0" -> "3.18.x"; + "3.17.0" -> "3.19.x" [color=red, style=dashed]; + "3.19.x" -> "3.20.x"; + "3.19.x" -> "3.21.x" [color=red]; + "3.18.x" -> debian_lenny; + "3.18.x" -> "3.20.x" [color=blue, style=dashed]; + "3.21.x" -> "3.21.11"; + "3.21.x" -> "4.1.0" [color=red]; + "3.21.x" -> "perf"; + "perf" -> "4.1.0"; + "4.1.0" -> "4.1.4" [color=red, style=dashed]; + "4.1.4" -> "file-errHdlr"; + "4.1.4" -> "4.1.5" [color=red]; + "4.1.5" -> "4.1.6" [color=red]; + "3.21.x" -> msgnolock + "3.21.x" -> "imudp-select"; + "4.1.5" -> solaris; + "file-errHdlr" -> "4.1.6"; + "tests" -> "4.1.6"; + + "1.0.5" -> v1stable [dir=none, style=dotted]; + "2.0.5" -> v2stable [dir=none, style=dotted]; + "3.20.x" -> v3stable [dir=none, style=dotted]; + "3.21.11" -> beta [dir=none, style=dotted]; + "4.1.6" -> devel [dir=none, style=dotted]; + + {rank=same; "4.1.5" "solaris"} + {rank=same; "3.18.x" "debian_lenny"} + {rank=same; "1.0.5" "2.0.5" "3.20.x" "3.21.11" "4.1.6"} +} -- cgit From 67e00c063122de13dd6c6354fa095978aa1773de Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 25 Mar 2009 17:59:35 +0100 Subject: bugfix: fixed some segaults on Solaris where vsprintf() does not check for NULL pointers --- ChangeLog | 2 ++ tools/omfwd.c | 1 - tools/syslogd.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0cbe7bba..1dbd69e0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,6 +19,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? - improved omfile so that it properly suspends itself if there is an i/o or file name generation error. This enables it to be used with the full high availability features of rsyslog's engine +- bugfix: fixed some segaults on Solaris, where vsprintf() does not + check for NULL pointers --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/tools/omfwd.c b/tools/omfwd.c index 1dd184ef..7a945ce0 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -615,7 +615,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) } else { CHKmalloc(pData->f_hname = strdup((char*) q)); } -dbgprintf("hostname '%s', port '%s'\n", pData->f_hname, pData->port); /* process template */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, diff --git a/tools/syslogd.c b/tools/syslogd.c index 235bc52e..a2aead9a 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3475,7 +3475,7 @@ int realMain(int argc, char **argv) /* END core initializations - we now come back to carrying out command line options*/ while((iRet = bufOptRemove(&ch, &arg)) == RS_RET_OK) { - dbgprintf("deque option %c, optarg '%s'\n", ch, arg); + dbgprintf("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); switch((char)ch) { case '4': glbl.SetDefPFFamily(PF_INET); -- cgit From 5103c912eeb8c0d410f641a971d3687a02c8c02b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 26 Mar 2009 15:30:46 +0100 Subject: parser test script created more or less complete now, with some minor nits left for later but is usable. --- tests/parser.tcl | 76 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test.tcl | 41 ------------------------- tests/testruns/parser.conf | 3 +- 3 files changed, 78 insertions(+), 42 deletions(-) create mode 100644 tests/parser.tcl delete mode 100644 tests/test.tcl diff --git a/tests/parser.tcl b/tests/parser.tcl new file mode 100644 index 00000000..274c08a3 --- /dev/null +++ b/tests/parser.tcl @@ -0,0 +1,76 @@ +# rsyslog parser tests +# This is a first version, and can be extended and improved for +# sure. But it is far better than nothing. Please note that this +# script works together with the config file AND easily extensible +# test case files (*.parse1) to run a number of checks. All test +# cases are executed, even if there is a failure early in the +# process. When finished, the numberof failed tests will be given. +# +# Note: a lot of things are not elegant, but at least they work... +# Even simple things seem to be somewhat non-simple if you are +# not sufficiently involved with tcl/expect ;) -- rgerhards +# +# Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH +# +# This file is part of rsyslog. + + + +# HELP HELP HELP HELP HELP HELP HELP HELP +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# If you happen to know how to disable rsyslog's +# stdout from appearing on the "real" stdout, please +# let me know. This is annouying, but I have no more +# time left to invest finding a solution (as the +# rest basically works well...). +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +package require Expect +package require udp 1.0 + +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; +#interact; +expect "}}"; # eat startup message +set udpSock [udp_open]; +udp_conf $udpSock 127.0.0.1 514 +set files [glob "testruns/*.parse1"] +set failed 0; +puts "\n"; + +set i 1; + +foreach testcase $files { + puts "testing $testcase ..."; + set fp [open "$testcase" r]; + fconfigure $fp -buffering line + gets $fp input + gets $fp expected + # assemble "expected" to match the template we use + close $fp; + + # send to daemon + puts $udpSock $input; + flush $udpSock; + + # get response and compare + expect -re "{{.*}}"; + puts "\n"; # at least we make the output readbale... + + set result $expect_out(buffer); + set result [string trimleft $result "\{\{"]; + set result [string trimright $result "\}\}"]; + + if { $result != $expected } { + puts "test $i failed!\n"; + puts "expected: '$expected'\n"; + puts "returned: '$result'\n"; + puts "\n"; + set failed [expr {$failed + 1}]; + }; + set i [expr {$i + 1}]; +} + +exec kill $rsyslogdPID; +close $udpSock; + +puts "Number of failed test: $failed.\n"; diff --git a/tests/test.tcl b/tests/test.tcl deleted file mode 100644 index 7f1b7a6b..00000000 --- a/tests/test.tcl +++ /dev/null @@ -1,41 +0,0 @@ -# rsyslog parser tests -package require Expect -package require udp 1.0 - -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; -#interact; -#puts "pid: $rsyslogdPID"; -#sleep 1; -#expect "\n"; -expect "}}"; # eat startup message -set udpSock [udp_open]; -udp_conf $udpSock 127.0.0.1 514 -set files [glob *.parse1] -puts "done init\n"; - - -foreach testcase $files { - puts "File $testcase"; - set fp [open "$testcase" r]; - fconfigure $fp -buffering line - #set data [read $fp]; - #set data [split $data "\n"]; - gets $fp input - puts "Line 1: $input\n"; - - puts $udpSock $input; - flush $udpSock; - - - set i 1 - expect -re "{{.*}}"; - set result $expect_out(buffer); - - #puts "MSG $i: '$expect_out(buffer)'"; - puts "MSG $i: '$result'\n"; - set i [expr {$i + 1}]; - -} - -exec kill $rsyslogdPID; -close $udpSock; diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index a515ff6c..3558f143 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -6,5 +6,6 @@ $UDPServerRun 514 $ErrorMessagesToStderr off # use a special format that we can easily parse in expect -$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" +#$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" +$template expect,"{{%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%}}" *.* :omstdout:;expect -- cgit From 0be199af6cc2cd9a9acb96e97e1cd061affaa6d9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 26 Mar 2009 17:48:59 +0100 Subject: initial parser testing suite added integrated tcl test script into autotools make check, created a few test cases based a real-world sample and samples from RFC3164 and 5424. --- ChangeLog | 2 ++ tests/Makefile.am | 10 +++++----- tests/parser.tcl | 3 ++- tests/parsertest | 2 ++ tests/testruns/1.parse1 | 3 +++ tests/testruns/rfc3164.parse1 | 4 ++++ tests/testruns/rfc5424-1.parse1 | 4 ++++ tests/testruns/rfc5424-2.parse1 | 4 ++++ tests/testruns/rfc5424-3.parse1 | 4 ++++ tests/testruns/rfc5424-4.parse1 | 4 ++++ 10 files changed, 34 insertions(+), 6 deletions(-) create mode 100755 tests/parsertest create mode 100644 tests/testruns/1.parse1 create mode 100644 tests/testruns/rfc3164.parse1 create mode 100644 tests/testruns/rfc5424-1.parse1 create mode 100644 tests/testruns/rfc5424-2.parse1 create mode 100644 tests/testruns/rfc5424-3.parse1 create mode 100644 tests/testruns/rfc5424-4.parse1 diff --git a/ChangeLog b/ChangeLog index 1cfb3c3a..e74c7209 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? is used in a secure way. (may be removed later) - added omstdout module for testing purposes. Spits out all messages to stdout - no config option, no other features +- added a parser testing suite (still needs to be extended, but a good + start) - modified $ModLoad statement so that for modules whom's name starts with a dot, no path is prepended (this enables relative-pathes and should not break any valid current config) diff --git a/tests/Makefile.am b/tests/Makefile.am index 384afd4e..225f087f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,5 @@ -#AUTOMAKE_OPTIONS=dejagnu -#DEJATOOL=Rainer - check_PROGRAMS = rt_init rscript -TESTS = $(check_PROGRAMS) cfg.sh +TESTS = $(check_PROGRAMS) cfg.sh parsertest TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ #TESTS = $(check_PROGRAMS) @@ -19,7 +16,10 @@ EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ cfg4.testin \ DevNull.cfgtest \ err1.rstest \ - NoExistFile.cfgtest + NoExistFile.cfgtest \ + parsertest + parser.tcl + testruns/*.parser1 rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/parser.tcl b/tests/parser.tcl index 274c08a3..6b08717b 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -73,4 +73,5 @@ foreach testcase $files { exec kill $rsyslogdPID; close $udpSock; -puts "Number of failed test: $failed.\n"; +puts "Number of failed tests: $failed.\n"; +if { $failed != 0 } { exit 1 }; diff --git a/tests/parsertest b/tests/parsertest new file mode 100755 index 00000000..78c42c07 --- /dev/null +++ b/tests/parsertest @@ -0,0 +1,2 @@ +# run parser test suite +tclsh parser.tcl diff --git a/tests/testruns/1.parse1 b/tests/testruns/1.parse1 new file mode 100644 index 00000000..5ae655e6 --- /dev/null +++ b/tests/testruns/1.parse1 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,local4,debug,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc3164.parse1 b/tests/testruns/rfc3164.parse1 new file mode 100644 index 00000000..e7a5fa18 --- /dev/null +++ b/tests/testruns/rfc3164.parse1 @@ -0,0 +1,4 @@ +<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine,su,su:, 'su root' failed for lonvick on /dev/pts/8 +#Example from RFC3164, section 5.4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-1.parse1 b/tests/testruns/rfc5424-1.parse1 new file mode 100644 index 00000000..90236c7f --- /dev/null +++ b/tests/testruns/rfc5424-1.parse1 @@ -0,0 +1,4 @@ +<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 +#Example from RFC5424, section 6.5 / sample 1 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-2.parse1 b/tests/testruns/rfc5424-2.parse1 new file mode 100644 index 00000000..a86fbc35 --- /dev/null +++ b/tests/testruns/rfc5424-2.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. +165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. +#Example from RFC5424, section 6.5 / sample 2 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-3.parse1 b/tests/testruns/rfc5424-3.parse1 new file mode 100644 index 00000000..6ad4073d --- /dev/null +++ b/tests/testruns/rfc5424-3.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"][examplePriority@32473 class="high"] +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog, +#Example from RFC5424, section 6.5 / sample 4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-4.parse1 b/tests/testruns/rfc5424-4.parse1 new file mode 100644 index 00000000..ecf27e14 --- /dev/null +++ b/tests/testruns/rfc5424-4.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"] BOMAn application event log entry... +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog,BOMAn application event log entry... +#Example from RFC5424, section 6.5 / sample 3 +#Only the first two lines are important, you may place anything behind them! -- cgit From 47fb9cb807d9c646cb07e56e380f3c553176955c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 26 Mar 2009 18:42:49 +0100 Subject: added some missing files in tests Makefile.am --- tests/Makefile.am | 12 ++++++++---- tests/parser.tcl | 23 ++++++----------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 225f087f..5b579017 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,10 +1,10 @@ check_PROGRAMS = rt_init rscript TESTS = $(check_PROGRAMS) cfg.sh parsertest TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ -#TESTS = $(check_PROGRAMS) test_files = testbench.h runtime-dummy.c -EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ +EXTRA_DIST=parser.tcl \ + 1.rstest 2.rstest 3.rstest err1.rstest \ cfg.sh \ cfg1.cfgtest \ cfg1.testin \ @@ -17,9 +17,13 @@ EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ + testruns/1.parse1 \ + testruns/rfc3164.parse1 \ + testruns/rfc5424-1.parse1 \ + testruns/rfc5424-2.parse1 \ + testruns/rfc5424-3.parse1 \ + testruns/rfc5424-4.parse1 \ parsertest - parser.tcl - testruns/*.parser1 rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/parser.tcl b/tests/parser.tcl index 6b08717b..5872fa3c 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -14,19 +14,9 @@ # # This file is part of rsyslog. - - -# HELP HELP HELP HELP HELP HELP HELP HELP -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# If you happen to know how to disable rsyslog's -# stdout from appearing on the "real" stdout, please -# let me know. This is annouying, but I have no more -# time left to invest finding a solution (as the -# rest basically works well...). -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - package require Expect package require udp 1.0 +log_user 0; # comment this out if you would like to see rsyslog output for testing set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; #interact; @@ -35,12 +25,12 @@ set udpSock [udp_open]; udp_conf $udpSock 127.0.0.1 514 set files [glob "testruns/*.parse1"] set failed 0; -puts "\n"; +puts "\nExecuting parser test suite..."; -set i 1; +set i 0; foreach testcase $files { - puts "testing $testcase ..."; + puts "testing $testcase"; set fp [open "$testcase" r]; fconfigure $fp -buffering line gets $fp input @@ -54,14 +44,13 @@ foreach testcase $files { # get response and compare expect -re "{{.*}}"; - puts "\n"; # at least we make the output readbale... set result $expect_out(buffer); set result [string trimleft $result "\{\{"]; set result [string trimright $result "\}\}"]; if { $result != $expected } { - puts "test $i failed!\n"; + puts "failed!"; puts "expected: '$expected'\n"; puts "returned: '$result'\n"; puts "\n"; @@ -73,5 +62,5 @@ foreach testcase $files { exec kill $rsyslogdPID; close $udpSock; -puts "Number of failed tests: $failed.\n"; +puts "Total number of tests: $i, number of failed tests: $failed.\n"; if { $failed != 0 } { exit 1 }; -- cgit From 581361121524bf66744e023b0a0c8ab448cfaa4a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 27 Mar 2009 16:09:45 +0100 Subject: fixed a couple of nits with "make [dist]check" --- tests/Makefile.am | 2 ++ tests/parser.tcl | 16 ++++++++++++++-- tests/parsertest | 2 +- tests/testruns/parser.conf | 2 -- tests/work/dummy | 3 +++ 5 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 tests/work/dummy diff --git a/tests/Makefile.am b/tests/Makefile.am index 5b579017..0de5a2f1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -17,12 +17,14 @@ EXTRA_DIST=parser.tcl \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ + testruns/parser.conf \ testruns/1.parse1 \ testruns/rfc3164.parse1 \ testruns/rfc5424-1.parse1 \ testruns/rfc5424-2.parse1 \ testruns/rfc5424-3.parse1 \ testruns/rfc5424-4.parse1 \ + work/dummy \ parsertest rt_init_SOURCES = rt-init.c $(test_files) diff --git a/tests/parser.tcl b/tests/parser.tcl index 5872fa3c..047607c6 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -9,6 +9,8 @@ # Note: a lot of things are not elegant, but at least they work... # Even simple things seem to be somewhat non-simple if you are # not sufficiently involved with tcl/expect ;) -- rgerhards +# +# call: tclsh parser.tcl /director/with/testcases # # Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH # @@ -18,12 +20,22 @@ package require Expect package require udp 1.0 log_user 0; # comment this out if you would like to see rsyslog output for testing -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; +if {$argc > 1} { + puts "invalid number of parameters, usage: tclsh parser.tcl /directory/with/testcases"; + exit 1; +} +if {$argc == 0 } { + set srcdir "."; +} else { + set srcdir "$argv"; +} + +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-i$srcdir/work/rsyslog.pid" "-M../runtime/.libs"]; #interact; expect "}}"; # eat startup message set udpSock [udp_open]; udp_conf $udpSock 127.0.0.1 514 -set files [glob "testruns/*.parse1"] +set files [glob "$srcdir/testruns/*.parse1"] set failed 0; puts "\nExecuting parser test suite..."; diff --git a/tests/parsertest b/tests/parsertest index 78c42c07..c7efa631 100755 --- a/tests/parsertest +++ b/tests/parsertest @@ -1,2 +1,2 @@ # run parser test suite -tclsh parser.tcl +tclsh $srcdir/parser.tcl $srcdir diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index 3558f143..7b4b4aed 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -1,11 +1,9 @@ $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imuxsock/.libs/imuxsock $ModLoad ../plugins/imudp/.libs/imudp $UDPServerRun 514 $ErrorMessagesToStderr off # use a special format that we can easily parse in expect -#$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" $template expect,"{{%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%}}" *.* :omstdout:;expect diff --git a/tests/work/dummy b/tests/work/dummy new file mode 100644 index 00000000..93c5babb --- /dev/null +++ b/tests/work/dummy @@ -0,0 +1,3 @@ +This is a dummy file. It's only purpose is to ensure +that ./test/work is created so that "make distcheck" +and "make check" can operate properly. -- cgit From 8e29c1fc47523c894b78894d6fdeb43f2d97811d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 27 Mar 2009 17:13:28 +0100 Subject: solved some more issues with "make [dist]check" especially when executed as non-root --- tests/Makefile.am | 2 +- tests/parser.tcl | 4 ++-- tests/testruns/parser.conf | 2 +- tests/work/dummy | 3 --- 4 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 tests/work/dummy diff --git a/tests/Makefile.am b/tests/Makefile.am index 0de5a2f1..f22ca139 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,7 @@ check_PROGRAMS = rt_init rscript TESTS = $(check_PROGRAMS) cfg.sh parsertest TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ +DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c EXTRA_DIST=parser.tcl \ @@ -24,7 +25,6 @@ EXTRA_DIST=parser.tcl \ testruns/rfc5424-2.parse1 \ testruns/rfc5424-3.parse1 \ testruns/rfc5424-4.parse1 \ - work/dummy \ parsertest rt_init_SOURCES = rt-init.c $(test_files) diff --git a/tests/parser.tcl b/tests/parser.tcl index 047607c6..1adeac25 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -30,11 +30,11 @@ if {$argc == 0 } { set srcdir "$argv"; } -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-i$srcdir/work/rsyslog.pid" "-M../runtime/.libs"]; +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-irsyslog.pid" "-M../runtime/.libs"]; #interact; expect "}}"; # eat startup message set udpSock [udp_open]; -udp_conf $udpSock 127.0.0.1 514 +udp_conf $udpSock 127.0.0.1 12514 set files [glob "$srcdir/testruns/*.parse1"] set failed 0; puts "\nExecuting parser test suite..."; diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index 7b4b4aed..8d32746c 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -1,6 +1,6 @@ $ModLoad ../plugins/omstdout/.libs/omstdout $ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 514 +$UDPServerRun 12514 $ErrorMessagesToStderr off diff --git a/tests/work/dummy b/tests/work/dummy deleted file mode 100644 index 93c5babb..00000000 --- a/tests/work/dummy +++ /dev/null @@ -1,3 +0,0 @@ -This is a dummy file. It's only purpose is to ensure -that ./test/work is created so that "make distcheck" -and "make check" can operate properly. -- cgit From 3e3a9bc9982331e44cf397fef131e75553f2ab2c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 12:00:40 +0200 Subject: ported non-tcl based test suite to Solaris --- configure.ac | 2 +- tests/Makefile.am | 12 +++++++----- tests/cfg.sh | 12 ++++++------ tests/ourtail.c | 43 +++++++++++++++++++++++++++++++++++++++++++ tests/rscript.c | 41 +++++++++++++++++++++++++++++++++++++---- 5 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 tests/ourtail.c diff --git a/configure.ac b/configure.ac index 0c9483ca..7289d74e 100644 --- a/configure.ac +++ b/configure.ac @@ -98,7 +98,7 @@ AC_TYPE_SIGNAL AC_FUNC_STAT AC_FUNC_STRERROR_R AC_FUNC_VPRINTF -AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait]) +AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline]) # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) diff --git a/tests/Makefile.am b/tests/Makefile.am index 7a31be45..65a294ca 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,7 +1,7 @@ -check_PROGRAMS = rt_init rscript -TESTS = $(check_PROGRAMS) cfg.sh +testruns = rt_init rscript +check_PROGRAMS = $(testruns) ourtail +TESTS = $(testruns) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ -#TESTS = $(check_PROGRAMS) test_files = testbench.h runtime-dummy.c EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ @@ -18,12 +18,14 @@ EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ err1.rstest \ NoExistFile.cfgtest +ourtail_SOURCES = ourtail.c + rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rt_init_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) +rt_init_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rt_init_LDFLAGS = -export-dynamic rscript_SOURCES = rscript.c $(test_files) rscript_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rscript_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) +rscript_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rscript_LDFLAGS = -export-dynamic diff --git a/tests/cfg.sh b/tests/cfg.sh index fb22fbf3..8acbf342 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -36,7 +36,7 @@ echo "local directory" # # check empty config file # -../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |./ourtail > tmp cmp tmp $srcdir/DevNull.cfgtest if [ ! $? -eq 0 ]; then echo "DevNull.cfgtest failed" @@ -51,7 +51,7 @@ fi; # # check missing config file # -../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |./ourtail > tmp cmp tmp $srcdir/NoExistFile.cfgtest if [ ! $? -eq 0 ]; then echo "NoExistFile.cfgtest failed" @@ -74,7 +74,7 @@ exit 0 # # check config with invalid directive # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg1.cfgtest if [ ! $? -eq 0 ]; then echo "cfg1.cfgtest failed" @@ -91,7 +91,7 @@ fi; # the one with the invalid config directive, so that we may see # an effect of the included config ;) # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg2.cfgtest if [ ! $? -eq 0 ]; then echo "cfg2.cfgtest failed" @@ -106,7 +106,7 @@ fi; # # check included config file, where included file does not exist # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg3.cfgtest if [ ! $? -eq 0 ]; then echo "cfg3.cfgtest failed" @@ -121,7 +121,7 @@ fi; # # check a reasonable complex, but correct, log file # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg4.cfgtest if [ ! $? -eq 0 ]; then echo "cfg4.cfgtest failed" diff --git a/tests/ourtail.c b/tests/ourtail.c new file mode 100644 index 00000000..f2751c72 --- /dev/null +++ b/tests/ourtail.c @@ -0,0 +1,43 @@ +/* This is a quick and dirty "tail implementation", one which always + * skips the first line, but nothing else. I have done this to prevent + * the various incompatible options of tail come into my way. One could + * probably work around this by using autoconf magic, but for me it + * was much quicker writing this small C program, which really should + * be portable across all platforms. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include + +int main(int argc, char *argv[]) +{ + int c; + + for(c = getchar() ; c != EOF && c != '\n' ; c = getchar()) + /*skip to newline*/; + + if(c == '\n') + c = getchar(); + + for( ; c != EOF ; c = getchar()) + putchar(c); +} diff --git a/tests/rscript.c b/tests/rscript.c index 3eec9c3c..4906f91a 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -39,6 +39,38 @@ DEFobjCurrIf(ctok) DEFobjCurrIf(ctok_token) DEFobjCurrIf(vmprg) + +/* we emulate getline (the dirty way) if we do not have it + * We do not try very hard, as this is just a test driver. + * rgerhards, 2009-03-31 + */ +#ifndef HAVE_GETLINE +ssize_t getline(char **lineptr, size_t *n, FILE *fp) +{ + int c; + int len = 0; + + if(*lineptr == NULL) + *lineptr = malloc(1024); /* quick and dirty! */ + + c = fgetc(fp); + while(c != EOF && c != '\n') { + (*lineptr)[len++] = c; + c = fgetc(fp); + } + if(c != EOF) /* need to add NL? */ + (*lineptr)[len++] = c; + + (*lineptr)[len] = '\0'; + + *n = len; + //printf("getline returns: '%s'\n", *lineptr); + + return (len > 0) ? len : -1; +} +#endif /* #ifndef HAVE_GETLINE */ + + BEGINInit CODESTARTInit pErrObj = "expr"; CHKiRet(objUse(expr, CORE_COMPONENT)); @@ -101,10 +133,9 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) CHKiRet(vmprg.Obj2Str(pExpr->pVmprg, pstrPrg)); if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { - int iLen; printf("error: compiled program different from expected result!\n"); - printf("generated vmprg (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); - printf("expected (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } @@ -139,6 +170,7 @@ ProcessTestFile(uchar *pszFileName) size_t lenLn; cstr_t *pstrIn = NULL; cstr_t *pstrOut = NULL; + int iParse; rsRetVal iRetExpected; DEFiRet; @@ -161,10 +193,11 @@ ProcessTestFile(uchar *pszFileName) /* once we had a comment, the next line MUST be "result: ". Anything * after nbr is simply ignored. */ - if(sscanf(lnptr, "result: %d", &iRetExpected) != 1) { + if(sscanf(lnptr, "result: %d", &iParse) != 1) { printf("error in result line, scanf failed, line: '%s'\n", lnptr); ABORT_FINALIZE(RS_RET_ERR); } + iRetExpected = iParse; getline(&lnptr, &lenLn, fp); CHKEOF; /* and now we look for "in:" (and again ignore the rest...) */ -- cgit From ec9e031599016b9eb6f9ac3fd8298ee4fcb0364f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 19:00:16 +0200 Subject: changed parser test suite to be c-program based I finally removed the tcl script because tcl costs a lot of time if you do not invest the full learning cycle, plus I have not everything avaible I need on Solaris. With C, I am quicker and I also can create a superior solution. So I finally switched. Took much less time than the initial tcl script... --- tests/Makefile.am | 10 +- tests/parser.tcl | 78 ------------ tests/parsertest | 2 - tests/parsertest.c | 265 ++++++++++++++++++++++++++++++++++++++++ tests/testruns/rfc5424-1.parse1 | 3 +- 5 files changed, 270 insertions(+), 88 deletions(-) delete mode 100644 tests/parser.tcl delete mode 100755 tests/parsertest create mode 100644 tests/parsertest.c diff --git a/tests/Makefile.am b/tests/Makefile.am index f22ca139..f941f3a6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,12 +1,10 @@ -check_PROGRAMS = rt_init rscript -TESTS = $(check_PROGRAMS) cfg.sh parsertest +check_PROGRAMS = rt_init rscript parsertest +TESTS = $(check_PROGRAMS) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c -EXTRA_DIST=parser.tcl \ - 1.rstest 2.rstest 3.rstest err1.rstest \ - cfg.sh \ +EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ cfg2.cfgtest \ @@ -25,7 +23,7 @@ EXTRA_DIST=parser.tcl \ testruns/rfc5424-2.parse1 \ testruns/rfc5424-3.parse1 \ testruns/rfc5424-4.parse1 \ - parsertest + cfg.sh rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/parser.tcl b/tests/parser.tcl deleted file mode 100644 index 1adeac25..00000000 --- a/tests/parser.tcl +++ /dev/null @@ -1,78 +0,0 @@ -# rsyslog parser tests -# This is a first version, and can be extended and improved for -# sure. But it is far better than nothing. Please note that this -# script works together with the config file AND easily extensible -# test case files (*.parse1) to run a number of checks. All test -# cases are executed, even if there is a failure early in the -# process. When finished, the numberof failed tests will be given. -# -# Note: a lot of things are not elegant, but at least they work... -# Even simple things seem to be somewhat non-simple if you are -# not sufficiently involved with tcl/expect ;) -- rgerhards -# -# call: tclsh parser.tcl /director/with/testcases -# -# Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH -# -# This file is part of rsyslog. - -package require Expect -package require udp 1.0 -log_user 0; # comment this out if you would like to see rsyslog output for testing - -if {$argc > 1} { - puts "invalid number of parameters, usage: tclsh parser.tcl /directory/with/testcases"; - exit 1; -} -if {$argc == 0 } { - set srcdir "."; -} else { - set srcdir "$argv"; -} - -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-irsyslog.pid" "-M../runtime/.libs"]; -#interact; -expect "}}"; # eat startup message -set udpSock [udp_open]; -udp_conf $udpSock 127.0.0.1 12514 -set files [glob "$srcdir/testruns/*.parse1"] -set failed 0; -puts "\nExecuting parser test suite..."; - -set i 0; - -foreach testcase $files { - puts "testing $testcase"; - set fp [open "$testcase" r]; - fconfigure $fp -buffering line - gets $fp input - gets $fp expected - # assemble "expected" to match the template we use - close $fp; - - # send to daemon - puts $udpSock $input; - flush $udpSock; - - # get response and compare - expect -re "{{.*}}"; - - set result $expect_out(buffer); - set result [string trimleft $result "\{\{"]; - set result [string trimright $result "\}\}"]; - - if { $result != $expected } { - puts "failed!"; - puts "expected: '$expected'\n"; - puts "returned: '$result'\n"; - puts "\n"; - set failed [expr {$failed + 1}]; - }; - set i [expr {$i + 1}]; -} - -exec kill $rsyslogdPID; -close $udpSock; - -puts "Total number of tests: $i, number of failed tests: $failed.\n"; -if { $failed != 0 } { exit 1 }; diff --git a/tests/parsertest b/tests/parsertest deleted file mode 100755 index c7efa631..00000000 --- a/tests/parsertest +++ /dev/null @@ -1,2 +0,0 @@ -# run parser test suite -tclsh $srcdir/parser.tcl $srcdir diff --git a/tests/parsertest.c b/tests/parsertest.c new file mode 100644 index 00000000..3ccae3d6 --- /dev/null +++ b/tests/parsertest.c @@ -0,0 +1,265 @@ +/* Runs a test suite on the rsyslog parser (and later potentially + * other things). + * + * Please note that this + * program works together with the config file AND easily extensible + * test case files (*.parse1) to run a number of checks. All test + * cases are executed, even if there is a failure early in the + * process. When finished, the numberof failed tests will be given. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 + + +void readLine(int fd, char *ln) +{ + char c; + int lenRead; + lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + *ln++ = c; + lenRead = read(fd, &c, 1); + } + *ln = '\0'; +} + + +/* send a message via UDP + * returns 0 if ok, something else otherwise. + */ +int +udpSend(char *buf, int lenBuf) +{ + struct sockaddr_in si_other; + int s, slen=sizeof(si_other); + + if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &si_other, 0, sizeof(si_other)); + si_other.sin_family = AF_INET; + si_other.sin_port = htons(12514); + if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + + if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { + perror("sendto"); + fprintf(stderr, "sendto() failed\n"); + return(1); + } + + close(s); + return 0; +} + +/* open pipe to test candidate - so far, this is + * always rsyslogd and with a fixed config. Later, we may + * change this. Returns 0 if ok, something else otherwise. + * rgerhards, 2009-03-31 + */ +int openPipe(pid_t *pid, int *pfd) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", + "-M../runtime//.libs", NULL }; + char confFile[1024]; + char *newenviron[] = { NULL }; + + + sprintf(confFile, "-f%s/testruns/parser2.conf", getenv("srcdir")); + newargv[1] = confFile; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if(cpid == 0) { /* Child reads from pipe */ + fclose(stdout); + dup(pipefd[1]); + close(pipefd[1]); + close(pipefd[0]); + fclose(stdin); + execve("../tools/rsyslogd", newargv, newenviron); + } else { + close(pipefd[1]); + *pid = cpid; + *pfd = pipefd[0]; + } + + return(0); +} + + +/* Process a specific test case. File name is provided. + * Needs to return 0 if all is OK, something else otherwise. + */ +int +processTestFile(int fd, char *pszFileName) +{ + FILE *fp; + char *testdata = NULL; + char *expected = NULL; + int ret = 0; + size_t lenLn; + char buf[4096]; + + if((fp = fopen((char*)pszFileName, "r")) == NULL) { + perror((char*)pszFileName); + return(2); + } + + /* skip comments at start of file */ + + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + + + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send */ + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + free(testdata); + free(expected); + fclose(fp); + return(ret); +} + + +/* carry out all tests. Tests are specified via a file name + * wildcard. Each of the files is read and the test carried + * out. + * Returns the number of tests that failed. Zero means all + * success. + */ +int +doTests(int fd, char *files) +{ + int iFailed = 0; + int iTests = 0; + int ret; + char *testFile; + glob_t testFiles; + size_t i = 0; + struct stat fileInfo; + + glob(files, GLOB_MARK, NULL, &testFiles); + + for(i = 0; i < testFiles.gl_pathc; i++) { + testFile = testFiles.gl_pathv[i]; + + if(stat((char*) testFile, &fileInfo) != 0) + continue; /* continue with the next file if we can't stat() the file */ + + ++iTests; + /* all regular files are run through the test logic. Symlinks don't work. */ + if(S_ISREG(fileInfo.st_mode)) { /* config file */ + printf("processing test case '%s' ... ", testFile); + ret = processTestFile(fd, testFile); + if(ret == 0) { + printf("successfully completed\n"); + } else { + printf("failed!\n"); + ++iFailed; + } + } + } + globfree(&testFiles); + + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + return(iFailed); +} + + +/* */ +int main(int argc, char *argv[]) +{ + int fd; + pid_t pid; + int ret = 0; + char buf[4096]; + char testcases[4096]; + + printf("running rsyslog parser tests ($srcdir=%s)\n", getenv("srcdir")); + + openPipe(&pid, &fd); + readLine(fd, buf); + + /* generate filename */ + sprintf(testcases, "%s/testruns/*.parse1", getenv("srcdir")); + if(doTests(fd, testcases) != 0) + ret = 1; + + /* cleanup */ + kill(pid, SIGTERM); + printf("End of parser tests.\n"); + exit(ret); +} + + diff --git a/tests/testruns/rfc5424-1.parse1 b/tests/testruns/rfc5424-1.parse1 index 90236c7f..23836c9f 100644 --- a/tests/testruns/rfc5424-1.parse1 +++ b/tests/testruns/rfc5424-1.parse1 @@ -1,4 +1,3 @@ +#Example from RFC5424, section 6.5 / sample 1 <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 -#Example from RFC5424, section 6.5 / sample 1 -#Only the first two lines are important, you may place anything behind them! -- cgit From 91d6888a8afe562bf4d2ef53be94c41898e1a2ec Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 22:03:02 +0200 Subject: bugfix: "make distcheck" did not work --- tests/parsertest.c | 12 ++++++++---- tests/testruns/parser.conf | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/parsertest.c b/tests/parsertest.c index 3ccae3d6..6c2221e8 100644 --- a/tests/parsertest.c +++ b/tests/parsertest.c @@ -105,7 +105,7 @@ int openPipe(pid_t *pid, int *pfd) char *newenviron[] = { NULL }; - sprintf(confFile, "-f%s/testruns/parser2.conf", getenv("srcdir")); + sprintf(confFile, "-f%s/testruns/parser.conf", getenv("srcdir")); newargv[1] = confFile; if (pipe(pipefd) == -1) { @@ -232,7 +232,13 @@ doTests(int fd, char *files) } globfree(&testFiles); - printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + if(iTests == 0) { + printf("Error: no test cases found, no tests executed.\n"); + iFailed = 1; + } else { + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + } + return(iFailed); } @@ -261,5 +267,3 @@ int main(int argc, char *argv[]) printf("End of parser tests.\n"); exit(ret); } - - diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index 8d32746c..0fb7d16d 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -5,5 +5,5 @@ $UDPServerRun 12514 $ErrorMessagesToStderr off # use a special format that we can easily parse in expect -$template expect,"{{%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%}}" +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" *.* :omstdout:;expect -- cgit From d27edc7587dba7b850759d151d90cdad1cb34a35 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 20:35:15 +0200 Subject: porting parser tests to solaris --- tests/Makefile.am | 12 ++++++++---- tests/getline.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/rscript.c | 31 ------------------------------ 3 files changed, 64 insertions(+), 35 deletions(-) create mode 100644 tests/getline.c diff --git a/tests/Makefile.am b/tests/Makefile.am index 093742db..09d1a0b6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,9 +1,10 @@ -check_PROGRAMS = rt_init rscript ourtail parsertest -TESTS = $(check_PROGRAMS) cfg.sh +TESTRUNS = rt_init rscript parsertest +check_PROGRAMS = $(TESTRUNS) ourtail +TESTS = $(TESTRUNS) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid - test_files = testbench.h runtime-dummy.c + EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ @@ -27,12 +28,15 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ ourtail_SOURCES = ourtail.c +parsertest_SOURCES = parsertest.c getline.c +parsertest_LDADD = $(SOL_LIBS) + rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) rt_init_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rt_init_LDFLAGS = -export-dynamic -rscript_SOURCES = rscript.c $(test_files) +rscript_SOURCES = rscript.c getline.c $(test_files) rscript_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) rscript_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rscript_LDFLAGS = -export-dynamic diff --git a/tests/getline.c b/tests/getline.c new file mode 100644 index 00000000..10de2ffe --- /dev/null +++ b/tests/getline.c @@ -0,0 +1,56 @@ +/* getline() replacement for platforms that do not have it. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include + +/* we emulate getline (the dirty way) if we do not have it + * We do not try very hard, as this is just a test driver. + * rgerhards, 2009-03-31 + */ +#ifndef HAVE_GETLINE +ssize_t getline(char **lineptr, size_t *n, FILE *fp) +{ + int c; + int len = 0; + + if(*lineptr == NULL) + *lineptr = malloc(4096); /* quick and dirty! */ + + c = fgetc(fp); + while(c != EOF && c != '\n') { + (*lineptr)[len++] = c; + c = fgetc(fp); + } + if(c != EOF) /* need to add NL? */ + (*lineptr)[len++] = c; + + (*lineptr)[len] = '\0'; + + *n = len; + //printf("getline returns: '%s'\n", *lineptr); + + return (len > 0) ? len : -1; +} +#endif /* #ifndef HAVE_GETLINE */ diff --git a/tests/rscript.c b/tests/rscript.c index 4906f91a..6b232f5f 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -40,37 +40,6 @@ DEFobjCurrIf(ctok_token) DEFobjCurrIf(vmprg) -/* we emulate getline (the dirty way) if we do not have it - * We do not try very hard, as this is just a test driver. - * rgerhards, 2009-03-31 - */ -#ifndef HAVE_GETLINE -ssize_t getline(char **lineptr, size_t *n, FILE *fp) -{ - int c; - int len = 0; - - if(*lineptr == NULL) - *lineptr = malloc(1024); /* quick and dirty! */ - - c = fgetc(fp); - while(c != EOF && c != '\n') { - (*lineptr)[len++] = c; - c = fgetc(fp); - } - if(c != EOF) /* need to add NL? */ - (*lineptr)[len++] = c; - - (*lineptr)[len] = '\0'; - - *n = len; - //printf("getline returns: '%s'\n", *lineptr); - - return (len > 0) ? len : -1; -} -#endif /* #ifndef HAVE_GETLINE */ - - BEGINInit CODESTARTInit pErrObj = "expr"; CHKiRet(objUse(expr, CORE_COMPONENT)); -- cgit From e3f21521cd6ae237fd9b362281fe188380210fb5 Mon Sep 17 00:00:00 2001 From: Demo Date: Tue, 31 Mar 2009 22:10:37 +0200 Subject: fixed some problems with "make check" interestingly, they manifested on Debian, only, but potentially existed on other platforms, too. --- configure.ac | 4 +++- tests/DevNull.cfgtest | 1 - tests/NoExistFile.cfgtest | 1 - tests/cfg.sh | 4 ++-- tools/omfile.c | 3 +-- tools/omfwd.c | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 2db856cd..de328f83 100644 --- a/configure.ac +++ b/configure.ac @@ -689,8 +689,10 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # settings for omstdout +# note that "make check" fails, if omstdout is not enabled (thus we enable +# it by default). AC_ARG_ENABLE(omstdout, - [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=yes@:>@])], [case "${enableval}" in yes) enable_omstdout="yes" ;; no) enable_omstdout="no" ;; diff --git a/tests/DevNull.cfgtest b/tests/DevNull.cfgtest index d30d936b..7822b6df 100644 --- a/tests/DevNull.cfgtest +++ b/tests/DevNull.cfgtest @@ -1,3 +1,2 @@ rsyslogd: CONFIG ERROR: there are no active actions configured. Inputs will run, but no output whatsoever is created. [try http://www.rsyslog.com/e/2103 ] rsyslogd: EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file! -rsyslogd: End of config validation run. Bye. diff --git a/tests/NoExistFile.cfgtest b/tests/NoExistFile.cfgtest index 4cbcc029..88d3123f 100644 --- a/tests/NoExistFile.cfgtest +++ b/tests/NoExistFile.cfgtest @@ -1,3 +1,2 @@ rsyslogd: CONFIG ERROR: could not interpret master config file '/This/does/not/exist'. [try http://www.rsyslog.com/e/2013 ] rsyslogd: EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file! -rsyslogd: End of config validation run. Bye. diff --git a/tests/cfg.sh b/tests/cfg.sh index 8acbf342..cb838354 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -36,7 +36,7 @@ echo "local directory" # # check empty config file # -../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |./ourtail > tmp +../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |./ourtail |head -2 > tmp cmp tmp $srcdir/DevNull.cfgtest if [ ! $? -eq 0 ]; then echo "DevNull.cfgtest failed" @@ -51,7 +51,7 @@ fi; # # check missing config file # -../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |./ourtail > tmp +../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |./ourtail |head -2 > tmp cmp tmp $srcdir/NoExistFile.cfgtest if [ ! $? -eq 0 ]; then echo "NoExistFile.cfgtest failed" diff --git a/tools/omfile.c b/tools/omfile.c index 65306846..5141b4da 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -725,7 +725,6 @@ CODESTARTparseSelectorAct } else { pData->bSyncFile = bEnableSync ? 1 : 0; } - pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */ switch (*p) @@ -822,7 +821,7 @@ CODESTARTparseSelectorAct if(pData->fd < 0 ) { pData->fd = -1; DBGPRINTF("Error opening log file: %s\n", pData->f_fname); - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); + errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); break; } if(strcmp((char*) p, _PATH_CONSOLE) == 0) diff --git a/tools/omfwd.c b/tools/omfwd.c index 7a945ce0..88a382e0 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -177,7 +177,7 @@ ENDfreeInstance BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - printf("%s", pData->f_hname); + dbgprintf("%s", pData->f_hname); ENDdbgPrintInstInfo -- cgit From a66b69533e38fdc40a7832b99c36e4d25bba80b2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 14:41:32 +0200 Subject: begining touches for putting rsyslog on spaceships --- doc/features.html | 1 + plugins/omdtn/Makefile.am | 8 +++ plugins/omdtn/omdtn.c | 130 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 plugins/omdtn/Makefile.am create mode 100644 plugins/omdtn/omdtn.c diff --git a/doc/features.html b/doc/features.html index 336b31cc..501f3304 100644 --- a/doc/features.html +++ b/doc/features.html @@ -124,6 +124,7 @@ community. Plus, it can be financially attractive: just think about how much les be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit of being recognised as a sponsor may even drive new customers to your business!
        +
      • Finalize the DTN "planetary Internet" space ship mode output plugin
      • port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge
      • pcre filtering - maybe (depending on feedback)  - diff --git a/plugins/omdtn/Makefile.am b/plugins/omdtn/Makefile.am new file mode 100644 index 00000000..afb57476 --- /dev/null +++ b/plugins/omdtn/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omdtn.la + +omdtn_la_SOURCES = omdtn.c +omdtn_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omdtn_la_LDFLAGS = -module -avoid-version +omdtn_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omdtn/omdtn.c b/plugins/omdtn/omdtn.c new file mode 100644 index 00000000..761bde79 --- /dev/null +++ b/plugins/omdtn/omdtn.c @@ -0,0 +1,130 @@ +/* omdtn.c + * This is the plugin for rsyslog use in the interplanetary Internet, + * especially useful for rsyslog in space ships of all kinds. + * The core idea was introduced in early 2009 and considered + * doable. + * + * Note that this has not yet been tested for robustness but needs + * to prior to placing it on top of a rocket. + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-04-01 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + write(1, (char*)ppString[0], strlen((char*)ppString[0])); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omstdout:", sizeof(":omstdout:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omstdout:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit + +/* vi:set ai: + */ -- cgit From 3954f2e166c3cbd78c71819c8d6c25120042dbcf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 12:48:07 +0200 Subject: added new "csv" property replacer option to enable simple creation of CSV-formatted outputs (format from RFC4180 is used) --- ChangeLog | 2 ++ doc/property_replacer.html | 14 +++++++++++++- runtime/msg.c | 34 ++++++++++++++++++++++++++++++++++ template.c | 5 +++++ template.h | 7 ++++--- 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 23a2e3e0..42fda5b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,8 @@ I only tweaked a very little bit. --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +- added new "csv" property replacer options to enable simple creation + of CSV-formatted outputs (format from RFC4180 is used) - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet diff --git a/doc/property_replacer.html b/doc/property_replacer.html index baf053f2..a6e9b518 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -314,6 +314,18 @@ case-insensitive. Currently, the following options are defined: convert property text to uppercase only +csv +formats the resulting field (after all modifications) in CSV format +as specified in RFC 4180. +Rsyslog will always use double quotes. Note that in order to have full CSV-formatted +text, you need to define a proper template. An example is this one: +
        $template csvline,"%syslogtag:::csv%,%msg:::csv%" +
        Most importantly, you need to provide the commas between the fields +inside the template. +
        This feature was introduced in rsyslog 4.1.6. + + + drop-last-lf The last LF in the message (if any), is dropped. Especially useful for PIX. @@ -411,7 +423,7 @@ syntax, this is where you actually use the property replacer.
        • [rsyslog site]

        This documentation is part of the rsyslog project.
        -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL version 2 or higher.

        diff --git a/runtime/msg.c b/runtime/msg.c index 9aa2ce84..5d1f21fd 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2405,6 +2405,40 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } } + /* finally, we need to check if the property should be formatted in CSV + * format (we use RFC 4180, and always use double quotes). As of this writing, + * this should be the last action carried out on the property, but in the + * future there may be reasons to change that. -- rgerhards, 2009-04-02 + */ + if(pTpe->data.field.options.bCSV) { + /* we need to obtain a private copy, as we need to at least add the double quotes */ + int iBufLen = strlen(pRes); + char *pBStart; + char *pDst; + char *pSrc; + /* the malloc may be optimized, we currently use the worst case... */ + pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(char)); + if(pDst == NULL) { + if(*pbMustBeFreed == 1) + free(pRes); + *pbMustBeFreed = 0; + return "**OUT OF MEMORY**"; + } + pSrc = pRes; + *pDst++ = '"'; /* starting quote */ + while(*pSrc) { + if(*pSrc == '"') + *pDst++ = '"'; /* need to add double double quote (see RFC4180) */ + *pDst++ = *pSrc++; + } + *pDst++ = '"'; /* ending quote */ + *pDst = '\0'; + if(*pbMustBeFreed == 1) + free(pRes); + pRes = pBStart; + *pbMustBeFreed = 1; + } + /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ return(pRes); } diff --git a/template.c b/template.c index 6fb7ba2b..4f507ff6 100644 --- a/template.c +++ b/template.c @@ -460,6 +460,8 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) pTpe->data.field.options.bSecPathDrop = 1; } else if(!strcmp((char*)Buf, "secpath-replace")) { pTpe->data.field.options.bSecPathReplace = 1; + } else if(!strcmp((char*)Buf, "csv")) { + pTpe->data.field.options.bCSV = 1; } else { dbgprintf("Invalid field option '%s' specified - ignored.\n", Buf); } @@ -1105,6 +1107,9 @@ void tplPrintList(void) if(pTpe->data.field.options.bSPIffNo1stSP) { dbgprintf("[SP iff no first SP] "); } + if(pTpe->data.field.options.bCSV) { + dbgprintf("[format as CSV (RFC4180)]"); + } if(pTpe->data.field.options.bDropLastLF) { dbgprintf("[drop last LF in msg] "); } diff --git a/template.h b/template.h index 04137b09..6ca8dc6c 100644 --- a/template.h +++ b/template.h @@ -94,9 +94,10 @@ struct templateEntry { unsigned bSpaceCC: 1; /* change control characters to spaceescape? */ unsigned bEscapeCC: 1; /* escape control characters? */ unsigned bDropLastLF: 1; /* drop last LF char in msg (PIX!) */ - unsigned bSecPathDrop: 1; /* drop slashes, replace dots, empty string */ - unsigned bSecPathReplace: 1; /* replace slashes, replace dots, empty string */ - unsigned bSPIffNo1stSP: 1; /* replace slashes, replace dots, empty string */ + unsigned bSecPathDrop: 1; /* drop slashes, replace dots, empty string */ + unsigned bSecPathReplace: 1; /* replace slashes, replace dots, empty string */ + unsigned bSPIffNo1stSP: 1; /* replace slashes, replace dots, empty string */ + unsigned bCSV: 1; /* format field in CSV (RFC 4180) format */ } options; /* options as bit fields */ } field; } data; -- cgit From 85c09e0c44c00be1cd91955b39fc6d0a17c72a98 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 12:57:37 +0200 Subject: add csv support to feature sheet --- doc/features.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/features.html b/doc/features.html index 501f3304..17a995bf 100644 --- a/doc/features.html +++ b/doc/features.html @@ -95,6 +95,9 @@ via custom plugins
      • an easy-to-write to plugin interface
      • ability to send SNMP trap messages
      • ability to filter out messages based on sequence of arrival
        • +
      • support for comma-seperated-values (CSV) output generation +(via the "csv" property replace option). The +CSV format supported is that from RFC 4180.
      • support for arbitrary complex boolean, string and arithmetic expressions in message filters
      -- cgit From eb807027af9e126a212b0630c5873dddae48963b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 15:12:57 +0200 Subject: added O_CLOEXEC to open() calls to make sure only the minimum number of file handles is left open during a exec call. This is not a 100% solution, as there are also some fopen() calls and, more importantly, file descriptors opened by libraries. But it is better than nothing (and it was quick, at least until we run into platform hell, what we will for sure ;)). --- plugins/imklog/linux.c | 2 +- runtime/debug.c | 2 +- tools/omfile.c | 12 ++++++------ tools/pidfile.c | 2 +- tools/syslogd.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/imklog/linux.c b/plugins/imklog/linux.c index 198b7c0e..0dd4320d 100644 --- a/plugins/imklog/linux.c +++ b/plugins/imklog/linux.c @@ -144,7 +144,7 @@ static enum LOGSRC GetKernelLogSrc(void) return(kernel); } - if ( (kmsg = open(_PATH_KLOG, O_RDONLY)) < 0 ) + if ( (kmsg = open(_PATH_KLOG, O_RDONLY|O_CLOEXEC)) < 0 ) { imklogLogIntMsg(LOG_ERR, "imklog: Cannot open proc file system, %d.\n", errno); ksyslog(7, NULL, 0); /* TODO: check this, implement more */ diff --git a/runtime/debug.c b/runtime/debug.c index 96004e47..4ee90226 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1343,7 +1343,7 @@ rsRetVal dbgClassInit(void) if(pszAltDbgFileName != NULL) { /* we have a secondary file, so let's open it) */ - if((altdbg = open(pszAltDbgFileName, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY, S_IRUSR|S_IWUSR)) == -1) { + if((altdbg = open(pszAltDbgFileName, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY|O_CLOEXEC, S_IRUSR|S_IWUSR)) == -1) { fprintf(stderr, "alternate debug file could not be opened, ignoring. Error: %s\n", strerror(errno)); } } diff --git a/tools/omfile.c b/tools/omfile.c index 5141b4da..36f160d1 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -301,7 +301,7 @@ int resolveFileSizeLimit(instanceData *pData) free(pCmd); - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); actualFileSize = lseek(pData->fd, 0, SEEK_END); @@ -394,13 +394,13 @@ prepareFile(instanceData *pData, uchar *newFileName) { DEFiRet; if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); + pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); FINALIZE; /* we are done in this case */ } if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); } else { pData->fd = -1; @@ -419,7 +419,7 @@ prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); if(pData->fd != -1) { /* check and set uid/gid */ @@ -653,7 +653,7 @@ again: && e == EBADF #endif ) { - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY); + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC); if (pData->fd < 0) { iRet = RS_RET_SUSPENDED; errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); @@ -742,7 +742,7 @@ CODESTARTparseSelectorAct pData->bDynamicName = 0; pData->fCreateMode = fCreateMode; /* preserve current setting */ pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); } break; diff --git a/tools/pidfile.c b/tools/pidfile.c index 2be13da6..e7744513 100644 --- a/tools/pidfile.c +++ b/tools/pidfile.c @@ -97,7 +97,7 @@ int write_pid (char *pidfile) int fd; int pid; - if ( ((fd = open(pidfile, O_RDWR|O_CREAT, 0644)) == -1) + if ( ((fd = open(pidfile, O_RDWR|O_CREAT|O_CLOEXEC, 0644)) == -1) || ((f = fdopen(fd, "r+")) == NULL) ) { fprintf(stderr, "Can't open or create %s.\n", pidfile); return 0; diff --git a/tools/syslogd.c b/tools/syslogd.c index c72160fb..16f255ea 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -549,7 +549,7 @@ void untty(void) int i; if ( !Debug ) { - i = open(_PATH_TTY, O_RDWR); + i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) (void) ioctl(i, (int) TIOCNOTTY, (char *)0); -- cgit From 8de35eaa2c9017df885dfd071a89288ca6a0e3af Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 15:37:17 +0200 Subject: clean compile on solaris --- runtime/cfsysline.c | 4 ++-- runtime/rsyslog.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index c4490b48..e1e4a6a4 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -364,7 +364,7 @@ static rsRetVal doGetGID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p /* we set value via a set function */ CHKiRet(pSetHdlr(pVal, pgBuf->gr_gid)); } - dbgprintf("gid %d obtained for group '%s'\n", pgBuf->gr_gid, szName); + dbgprintf("gid %d obtained for group '%s'\n", (int) pgBuf->gr_gid, szName); } skipWhiteSpace(pp); /* skip over any whitespace */ @@ -406,7 +406,7 @@ static rsRetVal doGetUID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p /* we set value via a set function */ CHKiRet(pSetHdlr(pVal, ppwBuf->pw_uid)); } - dbgprintf("uid %d obtained for user '%s'\n", ppwBuf->pw_uid, szName); + dbgprintf("uid %d obtained for user '%s'\n", (int) ppwBuf->pw_uid, szName); } skipWhiteSpace(pp); /* skip over any whitespace */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 899f5e13..032d8c04 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -341,6 +341,11 @@ typedef enum rsObjectID rsObjID; # define __attribute__(x) /*NOTHING*/ #endif +#ifndef O_CLOEXEC +/* of course, this limits the functionality... */ +# define O_CLOEXEC 0 +#endif + /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); -- cgit From a86e42028afeba1daca262b590bfd49d9c393b90 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 16:16:57 +0200 Subject: improved performance of regexp-based filters Thanks to Arnaud Cornet for providing the idea and initial patch. --- runtime/stringbuf.c | 30 ++++++++++++++++++++++++++++++ runtime/stringbuf.h | 2 ++ tools/syslogd.c | 8 ++++++-- tools/syslogd.h | 1 + 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index a5dc625a..c0a19ae4 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -724,6 +724,36 @@ finalize_it: RETiRet; } +/* same as above, only not braindead */ +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **rc) +{ + int ret; + + BEGINfunc + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + regex_t **cache = rc; + if (*cache == NULL) { + *cache = calloc(sizeof(regex_t), 1); + regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); + } + ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); + } else { + ret = 1; /* simulate "not found" */ + } + + ENDfunc + return ret; +} + +/* free a cached compiled regex */ +void rsRegexDestruct(void **rc) { + regex_t **cache = rc; + regexp.regfree(*cache); + free(*cache); + *cache = NULL; +} + /* compare a rsCStr object with a classical sz string. This function * is almost identical to rsCStrZsStrCmp(), but it also takes an offset diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index f3e08439..4b0fb065 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -137,6 +137,8 @@ int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **cache); +void rsRegexDestruct(void **rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); diff --git a/tools/syslogd.c b/tools/syslogd.c index 16f255ea..c7f36b45 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -432,6 +432,8 @@ selectorDestruct(void *pVal) } else if(pThis->f_filter_type == FILTER_EXPR) { if(pThis->f_filterData.f_expr != NULL) expr.Destruct(&pThis->f_filterData.f_expr); + if(pThis->regex_cache != NULL) + rsRegexDestruct(&pThis->regex_cache); } llDestroy(&pThis->llActList); @@ -1075,8 +1077,10 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce bRet = 1; /* process message! */ break; case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0) == RS_RET_OK) + //TODO REGEX: this needs to be merged with new functionality below + //rgerhards, 2009-04-02 + if(rsCStrSzStrMatchRegexCache(f->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, &f->regex_cache) == 0) bRet = 1; break; case FIOP_EREREGEX: diff --git a/tools/syslogd.h b/tools/syslogd.h index f1b11a91..ecaaec34 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -80,6 +80,7 @@ struct filed { } f_filterData; linkedList_t llActList; /* list of configured actions */ +regex_t *regex_cache; }; -- cgit From 1d16216aa326296673cc8520a8df351c4d492dfe Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 16:51:53 +0200 Subject: streamlined regex patch - abided to code conventions - fixed a potential segfault when regex library can not be loaded --- runtime/stringbuf.c | 61 ++++++++++++++++++++++++++++++++--------------------- runtime/stringbuf.h | 4 ++-- tools/syslogd.c | 6 +++--- tools/syslogd.h | 2 +- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index c0a19ae4..4a7cc4bd 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -725,33 +725,46 @@ finalize_it: } /* same as above, only not braindead */ -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **rc) +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *rc) { - int ret; - - BEGINfunc - - if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regex_t **cache = rc; - if (*cache == NULL) { - *cache = calloc(sizeof(regex_t), 1); - regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); - } - ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); - } else { - ret = 1; /* simulate "not found" */ - } - - ENDfunc - return ret; + int ret; + regex_t **cache = (regex_t**) rc; + + BEGINfunc + + assert(cache != NULL); + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + if (*cache == NULL) { + *cache = calloc(sizeof(regex_t), 1); + regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); + } + ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); + } else { + ret = 1; /* simulate "not found" */ + } + + ENDfunc + return ret; } -/* free a cached compiled regex */ -void rsRegexDestruct(void **rc) { - regex_t **cache = rc; - regexp.regfree(*cache); - free(*cache); - *cache = NULL; + +/* free a cached compiled regex + * Caller must provide a pointer to a buffer that was created by + * rsCStrSzStrMatchRegexCache() + */ +void rsCStrRegexDestruct(void *rc) +{ + regex_t **cache = rc; + + assert(cache != NULL); + assert(*cache != NULL); + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + regexp.regfree(*cache); + free(*cache); + *cache = NULL; + } } diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 4b0fb065..311d7f41 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -137,8 +137,8 @@ int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **cache); -void rsRegexDestruct(void **rc); +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *cache); +void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); diff --git a/tools/syslogd.c b/tools/syslogd.c index c7f36b45..d5429855 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -429,11 +429,11 @@ selectorDestruct(void *pVal) rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); if(pThis->f_filterData.prop.pCSCompValue != NULL) rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); + if(pThis->f_filterData.prop.regex_cache != NULL) + rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); } else if(pThis->f_filter_type == FILTER_EXPR) { if(pThis->f_filterData.f_expr != NULL) expr.Destruct(&pThis->f_filterData.f_expr); - if(pThis->regex_cache != NULL) - rsRegexDestruct(&pThis->regex_cache); } llDestroy(&pThis->llActList); @@ -1080,7 +1080,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce //TODO REGEX: this needs to be merged with new functionality below //rgerhards, 2009-04-02 if(rsCStrSzStrMatchRegexCache(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, &f->regex_cache) == 0) + (unsigned char*) pszPropVal, &f->f_filterData.prop.regex_cache) == 0) bRet = 1; break; case FIOP_EREREGEX: diff --git a/tools/syslogd.h b/tools/syslogd.h index ecaaec34..8b9bd131 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -73,6 +73,7 @@ struct filed { FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } operation; + regex_t *regex_cache; /* cache for compiled REs, if such are used */ cstr_t *pCSCompValue; /* value to "compare" against */ char isNegated; /* actually a boolean ;) */ } prop; @@ -80,7 +81,6 @@ struct filed { } f_filterData; linkedList_t llActList; /* list of configured actions */ -regex_t *regex_cache; }; -- cgit From 4ab540e3ba25a13fd079490ac52438e55dc92672 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 17:54:48 +0200 Subject: fully integrated regex patch Now have removed the previous method, as really nobody should call it any longer (and now nobody does ;)). Also did some other cleanup. --- ChangeLog | 2 ++ runtime/stringbuf.c | 41 +++++++++++++---------------------------- runtime/stringbuf.h | 3 +-- tools/syslogd.c | 8 +++----- 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/ChangeLog b/ChangeLog index 42fda5b3..fc9f807f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? the full high availability features of rsyslog's engine - bugfix: fixed some segaults on Solaris, where vsprintf() does not check for NULL pointers +- improved performance of regexp-based filters + Thanks to Arnaud Cornet for providing the idea and initial patch. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 4a7cc4bd..35ec44c6 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -703,49 +703,34 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) * never is a \0 *inside* a property string. * Note that the function returns -1 if regexp functionality is not available. * rgerhards: 2009-03-04: ERE support added, via parameter iType: 0 - BRE, 1 - ERE + * Arnaud Cornet/rgerhards: 2009-04-02: performance improvement by caching compiled regex + * If a caller does not need the cached version, it must still provide memory for it + * and must call rsCStrRegexDestruct() afterwards. */ -rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *rc) { - regex_t preq; + regex_t **cache = (regex_t**) rc; int ret; DEFiRet; - if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); - ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); - regexp.regfree(&preq); - if(ret != 0) - ABORT_FINALIZE(RS_RET_NOT_FOUND); - } else { - ABORT_FINALIZE(RS_RET_NOT_FOUND); - } - -finalize_it: - RETiRet; -} - -/* same as above, only not braindead */ -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *rc) -{ - int ret; - regex_t **cache = (regex_t**) rc; - - BEGINfunc - + assert(pCS1 != NULL); + assert(psz != NULL); assert(cache != NULL); if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { if (*cache == NULL) { *cache = calloc(sizeof(regex_t), 1); - regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); + regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); } ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); + if(ret != 0) + ABORT_FINALIZE(RS_RET_NOT_FOUND); } else { - ret = 1; /* simulate "not found" */ + ABORT_FINALIZE(RS_RET_NOT_FOUND); } - ENDfunc - return ret; +finalize_it: + RETiRet; } diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 311d7f41..684133bb 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -136,8 +136,7 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz); int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); -rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *cache); +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *cache); void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); diff --git a/tools/syslogd.c b/tools/syslogd.c index d5429855..b23c12a7 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1077,15 +1077,13 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce bRet = 1; /* process message! */ break; case FIOP_REGEX: - //TODO REGEX: this needs to be merged with new functionality below - //rgerhards, 2009-04-02 - if(rsCStrSzStrMatchRegexCache(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, &f->f_filterData.prop.regex_cache) == 0) + if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) bRet = 1; break; case FIOP_EREREGEX: if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1) == RS_RET_OK) + (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) bRet = 1; break; default: -- cgit From ec0e2c3e7df6addc02431628daddfeae49b92af7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 12:51:02 +0200 Subject: added a new way how output plugins may be passed parameters. This is more efficient for some outputs. They new can receive fields not only as a single string but rather in an array where each string is seperated. --- ChangeLog | 3 ++ action.c | 32 +++++++++++++++-- action.h | 2 ++ doc/rsyslog_conf_modules.html | 7 ++-- plugins/omstdout/omstdout.c | 80 +++++++++++++++++++++++++++++++++++++++++-- runtime/modules.c | 2 ++ runtime/objomsr.c | 18 ++++++++++ runtime/objomsr.h | 6 ++-- template.c | 56 ++++++++++++++++++++++++++++++ template.h | 1 + 10 files changed, 197 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index fc9f807f..7f79271f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -36,6 +36,9 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? check for NULL pointers - improved performance of regexp-based filters Thanks to Arnaud Cornet for providing the idea and initial patch. +- added a new way how output plugins may be passed parameters. This is + more effcient for some outputs. They new can receive fields not only + as a single string but rather in an array where each string is seperated. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/action.c b/action.c index 08755c13..03073153 100644 --- a/action.c +++ b/action.c @@ -425,6 +425,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) DEFiRet; int iRetries; int i; + int iArr; int iSleepPeriod; int bCallAction; int iCancelStateSave; @@ -439,7 +440,15 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + switch(pAction->eParamPassing) { + case ACT_STRING_PASSING: + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + break; + case ACT_ARRAY_PASSING: + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(ppMsgs[i]))); + break; + default:assert(0); /* software bug if this happens! */ + } } iRetries = 0; /* We now must guard the output module against execution by multiple threads. The @@ -495,7 +504,19 @@ finalize_it: /* cleanup */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { if(ppMsgs[i] != NULL) { - d_free(ppMsgs[i]); + switch(pAction->eParamPassing) { + case ACT_ARRAY_PASSING: + iArr = 0; + while(((char **)ppMsgs[i])[iArr] != NULL) + d_free(((char **)ppMsgs[i])[iArr++]); + d_free(ppMsgs[i]); + break; + case ACT_STRING_PASSING: + d_free(ppMsgs[i]); + break; + default: + assert(0); + } } } d_free(ppMsgs); @@ -905,6 +926,13 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques ABORT_FINALIZE(RS_RET_RQD_TPLOPT_MISSING); } + /* set parameter-passing mode */ + if(iTplOpts & OMSR_TPL_AS_ARRAY) { + pAction->eParamPassing = ACT_ARRAY_PASSING; + } else { + pAction->eParamPassing = ACT_STRING_PASSING; + } + dbgprintf("template: '%s' assigned\n", pTplName); } diff --git a/action.h b/action.h index dc9bbd74..f2706af6 100644 --- a/action.h +++ b/action.h @@ -61,6 +61,8 @@ struct action_s { short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ + enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1 } + eParamPassing; /* mode of parameter passing to action */ int iNumTpls; /* number of array entries for template element below */ struct template **ppTpl;/* array of template to use - strings must be passed to doAction * in this order. */ diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 890a55c8..a281d9e7 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -8,10 +8,9 @@ number of modules. Here is the entry point to their documentation and what they do (list is currently not complete)

        -
      • omsnmp - SNMP -trap output module
        • -
      • omrelp - RELP -output module
        • +
      • omsnmp - SNMP trap output module
        • +
      • omtdout - stdout output module (mainly a test tool)
        • +
      • omrelp - RELP output module
      • omgssapi - output module for GSS-enabled syslog
      • ommysql - output module for MySQL
      • ompgsql - output module for PostgreSQL
        • diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 6e227ba9..e491005c 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -49,7 +49,12 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA +/* config variables */ +static int bUseArrayInterface; /* shall action use array instead of string template interface? */ + + typedef struct _instanceData { + int bUseArrayInterface; /* uses action use array instead of string template interface? */ } instanceData; BEGINcreateInstance @@ -79,12 +84,46 @@ CODESTARTtryResume ENDtryResume BEGINdoAction + char **szParams; + char *toWrite; + int iParamVal; + int iParam; + int iBuf; + char szBuf[65564]; CODESTARTdoAction - write(1, (char*)ppString[0], strlen((char*)ppString[0])); + if(pData->bUseArrayInterface) { + /* if we use array passing, we need to put together a string + * ourselves. At this point, please keep in mind that omstdout is + * primarily a testing aid. Other modules may do different processing + * if they would like to support downlevel versions which do not support + * array-passing, but also use that interface on cores who do... + * So this code here is also more or less an example of how to do that. + * rgerhards, 2009-04-03 + */ + szParams = (char**) (ppString[0]); + /* In array-passing mode, ppString[] contains a NULL-terminated array + * of char *pointers. + */ + iParam = 0; + iBuf = 0; + while(szParams[iParam] != NULL) { + iParamVal = 0; + while(szParams[iParam][iParamVal] != '\0' && iBuf < sizeof(szBuf)) { + szBuf[iBuf++] = szParams[iParam][iParamVal++]; + } + ++iParam; + } + szBuf[iBuf] = '\0'; + toWrite = szBuf; + } else { + toWrite = (char*) ppString[0]; + } + write(1, toWrite, strlen(toWrite)); /* 1 is stdout! */ ENDdoAction BEGINparseSelectorAct + int iTplOpts; CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1) /* first check if this config line is actually for us */ @@ -99,7 +138,9 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) /* check if a non-standard template is to be applied */ if(*(p-1) == ';') --p; - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); + iTplOpts = (bUseArrayInterface == 0) ? 0 : OMSR_TPL_AS_ARRAY; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, iTplOpts, (uchar*) "RSYSLOG_FileFormat")); + pData->bUseArrayInterface = bUseArrayInterface; CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -115,10 +156,45 @@ CODEqueryEtryPt_STD_OMOD_QUERIES ENDqueryEtryPt + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + bUseArrayInterface = 0; + RETiRet; +} + + BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bArrayPassingSupported; /* does core support template passing as an array? */ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + /* check if the rsyslog core supports parameter passing code */ + bArrayPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + /* found entry point, so let's see if core supports array passing */ + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_ARRAY) + bArrayPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */ + } + DBGPRINTF("omstdout: array-passing is %ssupported by rsyslog core.\n", bArrayPassingSupported ? "" : "not "); + + if(bArrayPassingSupported) { + /* enable config comand only if core supports it */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomstdoutarrayinterface", 0, eCmdHdlrBinary, NULL, + &bUseArrayInterface, STD_LOADABLE_MODULE_ID)); + } + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit /* vi:set ai: diff --git a/runtime/modules.c b/runtime/modules.c index cef4eac6..9fdb48e7 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -225,6 +225,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) *pEtryPoint = regCfSysLineHdlr; } else if(!strcmp((char*) name, "objGetObjInterface")) { *pEtryPoint = objGetObjInterface; + } else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) { + *pEtryPoint = OMSRgetSupportedTplOpts; } else { *pEtryPoint = NULL; /* to be on the safe side */ ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND); diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 21d284f3..8dddc4b8 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -141,5 +141,23 @@ int OMSRgetEntry(omodStringRequest_t *pThis, int iEntry, uchar **ppTplName, int return RS_RET_OK; } + + +/* return the full set of template options that are supported by this version of + * OMSR. They are returned in an unsigned long value. The caller can mask that + * value to check on the option he is interested in. + * Note that this interface was added in 4.1.6, so a plugin must obtain a pointer + * to this interface via queryHostEtryPt(). + * rgerhards, 2009-04-03 + */ +rsRetVal +OMSRgetSupportedTplOpts(unsigned long *pOpts) +{ + DEFiRet; + assert(pOpts != NULL); + *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY; + RETiRet; +} + /* vim:set ai: */ diff --git a/runtime/objomsr.h b/runtime/objomsr.h index 2255e4f3..75ad0fb8 100644 --- a/runtime/objomsr.h +++ b/runtime/objomsr.h @@ -1,6 +1,6 @@ /* Definition of the omsr (omodStringRequest) object. * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -27,7 +27,8 @@ /* define flags for required template options */ #define OMSR_NO_RQD_TPL_OPTS 0 #define OMSR_RQD_TPL_OPT_SQL 1 -/* next option is 2, 4, 8, ... */ +#define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */ +/* next option is 4, 8, 16, ... */ struct omodStringRequest_s { /* strings requested by output module for doAction() */ int iNumEntries; /* number of array entries for data elements below */ @@ -40,6 +41,7 @@ typedef struct omodStringRequest_s omodStringRequest_t; rsRetVal OMSRdestruct(omodStringRequest_t *pThis); rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries); rsRetVal OMSRsetEntry(omodStringRequest_t *pThis, int iEntry, uchar *pTplName, int iTplOpts); +rsRetVal OMSRgetSupportedTplOpts(unsigned long *pOpts); int OMSRgetEntryCount(omodStringRequest_t *pThis); int OMSRgetEntry(omodStringRequest_t *pThis, int iEntry, uchar **ppTplName, int *piTplOpts); diff --git a/template.c b/template.c index 4f507ff6..4a12aa3b 100644 --- a/template.c +++ b/template.c @@ -47,6 +47,8 @@ static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ + + /* This functions converts a template into a string. It should * actually be in template.c, but this requires larger re-structuring * of the code (because all the property-access functions are static @@ -140,6 +142,60 @@ finalize_it: RETiRet; } + +/* This functions converts a template into an array of strings. + * For further general details, see the very similar funtion + * tpltoString(). + * Instead of a string, an array of string pointers is returned by + * thus function. The caller is repsonsible for destroying that array as + * well as all of its elements. The array is of fixed size. It's end + * is indicated by a NULL pointer. + * rgerhards, 2009-04-03 + */ +rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) +{ + DEFiRet; + struct templateEntry *pTpe; + uchar **pArr; + int iArr; + unsigned short bMustBeFreed; + uchar *pVal; + + assert(pTpl != NULL); + assert(pMsg != NULL); + assert(ppArr != NULL); + + /* loop through the template. We obtain one value, create a + * private copy (if necessary), add it to the string array + * and then on to the next until we have processed everything. + */ + + CHKmalloc(pArr = calloc(pTpl->tpenElements + 1, sizeof(uchar*))); + iArr = 0; + + pTpe = pTpl->pEntryRoot; + while(pTpe != NULL) { + if(pTpe->eEntryType == CONSTANT) { + CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); + } else if(pTpe->eEntryType == FIELD) { + pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); + if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ + pArr[iArr] = pVal; /* ... so we can use it! */ + } else { + CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pVal)); + } + } + iArr++; + pTpe = pTpe->pNext; + } + +finalize_it: + *ppArr = (iRet == RS_RET_OK) ? pArr : NULL; + + RETiRet; +} + + /* Helper to doSQLEscape. This is called if doSQLEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can diff --git a/template.h b/template.h index 6ca8dc6c..9d794f66 100644 --- a/template.h +++ b/template.h @@ -126,6 +126,7 @@ void tplLastStaticInit(struct template *tpl); * BEFORE msg.h, even if your code file does not actually need it. * rgerhards, 2007-08-06 */ +rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz); rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); -- cgit From 9103817f9fe811b49938036a1f9ff23672a9ec44 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 15:48:55 +0200 Subject: added (some) developer documentation for output plugin interface --- ChangeLog | 1 + doc/dev_oplugins.html | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/manual.html | 6 +- doc/status.html | 12 ++-- 4 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 doc/dev_oplugins.html diff --git a/ChangeLog b/ChangeLog index 7f79271f..8d4191d3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -39,6 +39,7 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? - added a new way how output plugins may be passed parameters. This is more effcient for some outputs. They new can receive fields not only as a single string but rather in an array where each string is seperated. +- added (some) developer documentation for output plugin interface --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html new file mode 100644 index 00000000..5bfc974c --- /dev/null +++ b/doc/dev_oplugins.html @@ -0,0 +1,178 @@ + + +writing rsyslog output plugins (developer's guide) + + +

        Writing Rsyslog Output Plugins

        +

        This page is the begin of some developer documentation for writing output +plugins. Doing so is quite easy (and that was a design goal), but there currently +is only sparse documentation on the process available. I was tempted NOT to +write this guide here because I know I will most probably not be able to +write a complete guide. +

        However, I finally concluded that it may be better to have same information +and pointers than to have nothing. +

        Getting Started and Samples

        +

        The best to get started with rsyslog plugin development is by looking at +existing plugins. All that start with "om" are output modules. That +means they are primarily thought of being message sinks. In theory, however, output +plugins may aggergate other functionality, too. Nobody has taken this route so far +so if you would like to do that, it is highly suggested to post your plan on the +rsyslog mailing list, first (so that we can offer advise). +

        The rsyslog distribution tarball contains two plugins that are extremely well +targeted for getting started: +

          +
        • omtemplate +
        • omstdout +
        +Plugin omtemplate was specifically created to provide a copy template for new output +plugins. It is bare of real functionality but has ample comments. Even if you decide +to start from another plugin (or even from scratch), be sure to read omtemplate source +and comments first. The omstdout is primarily a testing aide, but offers support for +the two different parameter-passing conventions plugins can use (plus the way to +differentiate between the two). It also is not bare of functionaly, only mostly +bare of it ;). But you can actually execute it and play with it. +

        In any case, you should also read the comments in ./runtime/module-template.h. +Output plugins are build based on a large set of code-generating macros. These +macros handle most of the plumbing needed by the interface. As long as no +special callback to rsyslog is needed (it typically is not), an output plugin does +not really need to be aware that it is executed by rsyslog. As a plug-in programmer, +you can (in most cases) "code as usual". However, all macros and entry points need to be +provided and thus reading the code comments in the files mentioned is highly suggested. +

        In short, the best idea is to start with a template. Let's assume you start by +copying omtemplate. Then, the basic steps you need to do are: +

          +
        • cp ./plugins/omtemplate ./plugins/your-plugin +
        • mv cd ./plugins/your-plugin +
        • vi Makefile.am, adjust to your-plugin +
        • mv omtemplate.c your-plugin.c +
        • cd ../.. +
        • vi Makefile.am configure.ac +
          serach for omtemplate, copy and modify (follow comments) +
        +

        Basically, this is all you need to do ... Well, except, of course, coding +your plugin ;). For testing, you need rsyslog's debugging support. Some useful +information is given in "troubleshooting rsyslog +from the doc set. +

        Special Topics

        +

        Threading

        +

        Rsyslog uses massive parallel processing and multithreading. However, a plugin's entry +points are guaranteed to be never called concurrently for the same action. +That means your plugin must be able to be called concurrently by two or more +threads, but you can be sure that for the same instance no concurrent calls +happen. This is guaranteed by the interface specification and the rsyslog core +guards against multiple concurrent calls. An instance, in simple words, is one +that shares a single instanceData structure. +

        So as long as you do not mess around with global data, you do not need +to think about multithreading (and can apply a purely sequential programming +methodology). +

        Please note that duringt the configuraton parsing stage of execution, access to +global variables for the configuration system is safe. In that stage, the core will +only call sequentially into the plugin. +

        Getting Message Data

        +

        The doAction() entry point of your plugin is provided with messages to be processed. +It will only be activated after filtering and all other conditions, so you do not need +to apply any other conditional but can simply process the message. +

        Note that you do NOT receive the full internal representation of the message +object. There are various (including historical) reasons for this and, among +others, this is a design decision based on security. +

        Your plugin will only receive what the end user has configured in a $template +statement. However, starting with 4.1.6, there are two ways of receiving the +template content. The default mode, and in most cases sufficient and optimal, +is to receive a single string with the expanded template. As I said, this is usually +optimal, think about writing things to files, emailing content or forwarding it. +

        The important philosophy is that a plugin should never reformat any +of such strings - that would either remove the user's ability to fully control +message formats or it would lead to duplicating code that is already present in the +core. If you need some formatting that is not yet present in the core, suggest it +to the rsyslog project, best done by sending a patch ;), and we will try hard to +get it into the core (so far, we could accept all such suggestions - no promise, though). +

        If a single string seems not suitable for your application, the plugin can also +request access to the template components. The typical use case seems to be databases, where +you would like to access properties via specific fields. With that mode, you receive a +char ** array, where each array element points to one field from the template (from +left to right). Fields start at arrray index 0 and a NULL pointer means you have +reached the end of the array (the typical Unix "poor man's linked list in an array" +design). Note, however, that each of the individual components is a string. It is +not a date stamp, number or whatever, but a string. This is because rsyslog processes +strings (from a high-level design look at it) and so this is the natural data type. +Feel free to convert to whatever you need, but keep in mind that malformed packets +may have lead to field contents you'd never expected... +

        If you like to use the array-based parameter passing method, think that it +is only available in rsyslog 4.1.6 and above. If you can accept that your plugin +will not be working with previous versions, you do not need to handle pre 4.1.6 cases. +However, it would be "nice" if you shut down yourself in these cases - otherwise the +older rsyslog core engine will pass you a string where you expect the array of pointers, +what most probably results in a segfault. To check whether or not the core supports the +functionality, you can use this code sequence: +

        +
+BEGINmodInit()
+	rsRetVal localRet;
+	rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts);
+	unsigned long opts;
+	int bArrayPassingSupported;		/* does core support template passing as an array? */
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	/* check if the rsyslog core supports parameter passing code */
+	bArrayPassingSupported = 0;
+	localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts);
+	if(localRet == RS_RET_OK) {
+		/* found entry point, so let's see if core supports array passing */
+		CHKiRet((*pomsrGetSupportedTplOpts)(&opts));
+		if(opts & OMSR_TPL_AS_ARRAY)
+			bArrayPassingSupported = 1;
+	} else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) {
+		ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */
+	}
+	DBGPRINTF("omstdout: array-passing is %ssupported by rsyslog core.\n", bArrayPassingSupported ? "" : "not ");
+
+	if(!bArrayPassingSupported) {
+		DBGPRINTF("rsyslog core too old, shutting down this plug-in\n");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+
+
        +

        The code first checks if the core supports the OMSRgetSupportedTplOpts() API (which is +also not present in all versions!) and, if so, queries the core if the OMSR_TPL_AS_ARRAY mode +is supported. If either does not exits, the core is too old for this functionality. The sample +snippet above then shuts down, but a plugin may instead just do things different. In +omstdout, you can see how a plugin may deal with the situation. +

        In any case, it is recommended that at least a graceful shutdown is made and the +array-passing capability not blindly be used. In such cases, we can not guard the +plugin from segfaulting and if the plugin (as currently always) is run within +rsyslog's process space, that results in a segfault for rsyslog. So do not do this. +

        Batching of Messages

        +

        With the current plugin interface, each message is passed via a separate call to the plugin. +This is annoying and costs performance in some uses cases (primarily for database outputs). +However, that's the way it (currently) is, no easy way around it. There are some ideas +to implement batching capabilities inside the rsyslog core, but without that the only +resort is to do it inside your plugin yourself. You are not prohibited from doing so. +There are some consequences, though: most importantly, the rsyslog core is no longer +intersted in messages that it passed to a plugin. As such, it will not try to make sure +the message is not lost before it was ultimately processed (because rsyslog, due to +doAction() returning successfully, thinks the message *was* ultimately processed). +

        When the rsyslog core receives batching capabilities, this will be implemented in +a way that is fully compatible to the existing plugin interface. While we have not yet +thought about the implementation, that will probably mean that some new interfaces +or options be used to turn on batching capabilities. +

        Licensing

        +

        From the rsyslog point of view, plugins constitute separate projects. As such, +we think plugins are not required to be compatible with GPLv3. However, this is +no legal advise. If you intend to release something under a non-GPLV3 compatible license +it is probably best to consult with your lawyer. +

        Most importantly, and this is definite, the rsyslog team does not expect +or require you to contribute your plugin to the rsyslog project (but of course +we are happy if you do). +

        Copyright

        +

        Copyright (c) 2009 Rainer Gerhards +and Adiscon.

        +

        Permission is granted to copy, distribute and/or modify this document under +the terms of the GNU Free Documentation License, Version 1.2 or any later +version published by the Free Software Foundation; with no Invariant Sections, +no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be +viewed at +http://www.gnu.org/copyleft/fdl.html.

        + + diff --git a/doc/manual.html b/doc/manual.html index 99a90594..0b03a255 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -64,7 +64,11 @@ the syslog priority (severity and facility) to the log file syslog sender over NAT (online only)
      • an overview and howto of rsyslog gssapi support
      • debug support in rsyslog
        • -
      • the rsyslog message queue object (developer's view)
        • +
      • Developer Documentation +

      Our rsyslog history page is for you if you would like to learn a little more diff --git a/doc/status.html b/doc/status.html index 59fd0809..dae94884 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,19 +2,19 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-02-02.

      +

      This page reflects the status as of 2009-04-03.

      Current Releases

      development: 4.1.5 [2009-03-11] - change log - download -
      beta: 3.21.10 [2009-02-02] - -change log - -download

      +
      beta: 3.21.11 [2009-04-03] - +change log - +download

      -

      v3 stable: 3.20.3 [2009-01-19] - change log - -download +

      v3 stable: 3.20.3 [2009-04-02] - change log - +download
      v2 stable: 2.0.6 [2008-08-07] - change log - download -- cgit From ce6b7e86cdd63ba1540d20aa22403d2b13d2e59f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 17:54:09 +0200 Subject: improved test suite and added test for new output module interface The testbench has now a generic driver that can run a whole class of test suites just by providing a config file and test cases. This does not cover all testing needs, but a lot. We have now added one test for the new array-passing output plugin interface. --- plugins/omstdout/omstdout.c | 2 + tests/Makefile.am | 28 ++-- tests/omod-if-array.sh | 1 + tests/parsertest.c | 269 --------------------------------- tests/parsertest.sh | 1 + tests/testruns/1.parse1 | 3 - tests/testruns/parser.conf | 9 -- tests/testruns/rfc3164.parse1 | 4 - tests/testruns/rfc5424-1.parse1 | 3 - tests/testruns/rfc5424-2.parse1 | 4 - tests/testruns/rfc5424-3.parse1 | 4 - tests/testruns/rfc5424-4.parse1 | 4 - tests/testsuites/1.omod-if-array | 2 + tests/testsuites/1.parse1 | 3 + tests/testsuites/omod-if-array.conf | 14 ++ tests/testsuites/parse1.conf | 9 ++ tests/testsuites/rfc3164.parse1 | 4 + tests/testsuites/rfc5424-1.parse1 | 3 + tests/testsuites/rfc5424-2.parse1 | 4 + tests/testsuites/rfc5424-3.parse1 | 4 + tests/testsuites/rfc5424-4.parse1 | 4 + tests/udptester.c | 291 ++++++++++++++++++++++++++++++++++++ 22 files changed, 358 insertions(+), 312 deletions(-) create mode 100755 tests/omod-if-array.sh delete mode 100644 tests/parsertest.c create mode 100755 tests/parsertest.sh delete mode 100644 tests/testruns/1.parse1 delete mode 100644 tests/testruns/parser.conf delete mode 100644 tests/testruns/rfc3164.parse1 delete mode 100644 tests/testruns/rfc5424-1.parse1 delete mode 100644 tests/testruns/rfc5424-2.parse1 delete mode 100644 tests/testruns/rfc5424-3.parse1 delete mode 100644 tests/testruns/rfc5424-4.parse1 create mode 100644 tests/testsuites/1.omod-if-array create mode 100644 tests/testsuites/1.parse1 create mode 100644 tests/testsuites/omod-if-array.conf create mode 100644 tests/testsuites/parse1.conf create mode 100644 tests/testsuites/rfc3164.parse1 create mode 100644 tests/testsuites/rfc5424-1.parse1 create mode 100644 tests/testsuites/rfc5424-2.parse1 create mode 100644 tests/testsuites/rfc5424-3.parse1 create mode 100644 tests/testsuites/rfc5424-4.parse1 create mode 100644 tests/udptester.c diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index e491005c..7c63b5c4 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -107,6 +107,8 @@ CODESTARTdoAction iParam = 0; iBuf = 0; while(szParams[iParam] != NULL) { + if(iParam > 0) + szBuf[iBuf++] = ','; /* all but first need a delimiter */ iParamVal = 0; while(szParams[iParam][iParamVal] != '\0' && iBuf < sizeof(szBuf)) { szBuf[iBuf++] = szParams[iParam][iParamVal++]; diff --git a/tests/Makefile.am b/tests/Makefile.am index 09d1a0b6..ab1c5a62 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ -TESTRUNS = rt_init rscript parsertest -check_PROGRAMS = $(TESTRUNS) ourtail -TESTS = $(TESTRUNS) cfg.sh +TESTRUNS = rt_init rscript +check_PROGRAMS = $(TESTRUNS) ourtail udptester +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -17,19 +17,23 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ - testruns/parser.conf \ - testruns/1.parse1 \ - testruns/rfc3164.parse1 \ - testruns/rfc5424-1.parse1 \ - testruns/rfc5424-2.parse1 \ - testruns/rfc5424-3.parse1 \ - testruns/rfc5424-4.parse1 \ + testsuites/parse1.conf \ + testsuites/1.parse1 \ + testsuites/rfc3164.parse1 \ + testsuites/rfc5424-1.parse1 \ + testsuites/rfc5424-2.parse1 \ + testsuites/rfc5424-3.parse1 \ + testsuites/rfc5424-4.parse1 \ + testsuites/omod-if-array.conf \ + testsuites/1.omod-if-array \ + parsertest.sh \ + omod-if-array.sh \ cfg.sh ourtail_SOURCES = ourtail.c -parsertest_SOURCES = parsertest.c getline.c -parsertest_LDADD = $(SOL_LIBS) +udptester_SOURCES = udptester.c getline.c +udptester_LDADD = $(SOL_LIBS) rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh new file mode 100755 index 00000000..cac08928 --- /dev/null +++ b/tests/omod-if-array.sh @@ -0,0 +1 @@ +./udptester omod-if-array diff --git a/tests/parsertest.c b/tests/parsertest.c deleted file mode 100644 index 6c2221e8..00000000 --- a/tests/parsertest.c +++ /dev/null @@ -1,269 +0,0 @@ -/* Runs a test suite on the rsyslog parser (and later potentially - * other things). - * - * Please note that this - * program works together with the config file AND easily extensible - * test case files (*.parse1) to run a number of checks. All test - * cases are executed, even if there is a failure early in the - * process. When finished, the numberof failed tests will be given. - * - * Part of the testbench for rsyslog. - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define EXIT_FAILURE 1 - - -void readLine(int fd, char *ln) -{ - char c; - int lenRead; - lenRead = read(fd, &c, 1); - while(lenRead == 1 && c != '\n') { - *ln++ = c; - lenRead = read(fd, &c, 1); - } - *ln = '\0'; -} - - -/* send a message via UDP - * returns 0 if ok, something else otherwise. - */ -int -udpSend(char *buf, int lenBuf) -{ - struct sockaddr_in si_other; - int s, slen=sizeof(si_other); - - if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { - perror("socket()"); - return(1); - } - - memset((char *) &si_other, 0, sizeof(si_other)); - si_other.sin_family = AF_INET; - si_other.sin_port = htons(12514); - if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { - fprintf(stderr, "inet_aton() failed\n"); - return(1); - } - - if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { - perror("sendto"); - fprintf(stderr, "sendto() failed\n"); - return(1); - } - - close(s); - return 0; -} - -/* open pipe to test candidate - so far, this is - * always rsyslogd and with a fixed config. Later, we may - * change this. Returns 0 if ok, something else otherwise. - * rgerhards, 2009-03-31 - */ -int openPipe(pid_t *pid, int *pfd) -{ - int pipefd[2]; - pid_t cpid; - char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime//.libs", NULL }; - char confFile[1024]; - char *newenviron[] = { NULL }; - - - sprintf(confFile, "-f%s/testruns/parser.conf", getenv("srcdir")); - newargv[1] = confFile; - - if (pipe(pipefd) == -1) { - perror("pipe"); - exit(EXIT_FAILURE); - } - - cpid = fork(); - if (cpid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } - - if(cpid == 0) { /* Child reads from pipe */ - fclose(stdout); - dup(pipefd[1]); - close(pipefd[1]); - close(pipefd[0]); - fclose(stdin); - execve("../tools/rsyslogd", newargv, newenviron); - } else { - close(pipefd[1]); - *pid = cpid; - *pfd = pipefd[0]; - } - - return(0); -} - - -/* Process a specific test case. File name is provided. - * Needs to return 0 if all is OK, something else otherwise. - */ -int -processTestFile(int fd, char *pszFileName) -{ - FILE *fp; - char *testdata = NULL; - char *expected = NULL; - int ret = 0; - size_t lenLn; - char buf[4096]; - - if((fp = fopen((char*)pszFileName, "r")) == NULL) { - perror((char*)pszFileName); - return(2); - } - - /* skip comments at start of file */ - - getline(&testdata, &lenLn, fp); - while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } - - - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send */ - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ - - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; - } - - free(testdata); - free(expected); - fclose(fp); - return(ret); -} - - -/* carry out all tests. Tests are specified via a file name - * wildcard. Each of the files is read and the test carried - * out. - * Returns the number of tests that failed. Zero means all - * success. - */ -int -doTests(int fd, char *files) -{ - int iFailed = 0; - int iTests = 0; - int ret; - char *testFile; - glob_t testFiles; - size_t i = 0; - struct stat fileInfo; - - glob(files, GLOB_MARK, NULL, &testFiles); - - for(i = 0; i < testFiles.gl_pathc; i++) { - testFile = testFiles.gl_pathv[i]; - - if(stat((char*) testFile, &fileInfo) != 0) - continue; /* continue with the next file if we can't stat() the file */ - - ++iTests; - /* all regular files are run through the test logic. Symlinks don't work. */ - if(S_ISREG(fileInfo.st_mode)) { /* config file */ - printf("processing test case '%s' ... ", testFile); - ret = processTestFile(fd, testFile); - if(ret == 0) { - printf("successfully completed\n"); - } else { - printf("failed!\n"); - ++iFailed; - } - } - } - globfree(&testFiles); - - if(iTests == 0) { - printf("Error: no test cases found, no tests executed.\n"); - iFailed = 1; - } else { - printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); - } - - return(iFailed); -} - - -/* */ -int main(int argc, char *argv[]) -{ - int fd; - pid_t pid; - int ret = 0; - char buf[4096]; - char testcases[4096]; - - printf("running rsyslog parser tests ($srcdir=%s)\n", getenv("srcdir")); - - openPipe(&pid, &fd); - readLine(fd, buf); - - /* generate filename */ - sprintf(testcases, "%s/testruns/*.parse1", getenv("srcdir")); - if(doTests(fd, testcases) != 0) - ret = 1; - - /* cleanup */ - kill(pid, SIGTERM); - printf("End of parser tests.\n"); - exit(ret); -} diff --git a/tests/parsertest.sh b/tests/parsertest.sh new file mode 100755 index 00000000..e7985bb0 --- /dev/null +++ b/tests/parsertest.sh @@ -0,0 +1 @@ +./udptester parse1 diff --git a/tests/testruns/1.parse1 b/tests/testruns/1.parse1 deleted file mode 100644 index 5ae655e6..00000000 --- a/tests/testruns/1.parse1 +++ /dev/null @@ -1,3 +0,0 @@ -<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 -167,local4,debug,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf deleted file mode 100644 index 0fb7d16d..00000000 --- a/tests/testruns/parser.conf +++ /dev/null @@ -1,9 +0,0 @@ -$ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 - -$ErrorMessagesToStderr off - -# use a special format that we can easily parse in expect -$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" -*.* :omstdout:;expect diff --git a/tests/testruns/rfc3164.parse1 b/tests/testruns/rfc3164.parse1 deleted file mode 100644 index e7a5fa18..00000000 --- a/tests/testruns/rfc3164.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 -34,auth,crit,Oct 11 22:14:15,mymachine,su,su:, 'su root' failed for lonvick on /dev/pts/8 -#Example from RFC3164, section 5.4 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-1.parse1 b/tests/testruns/rfc5424-1.parse1 deleted file mode 100644 index 23836c9f..00000000 --- a/tests/testruns/rfc5424-1.parse1 +++ /dev/null @@ -1,3 +0,0 @@ -#Example from RFC5424, section 6.5 / sample 1 -<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 -34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 diff --git a/tests/testruns/rfc5424-2.parse1 b/tests/testruns/rfc5424-2.parse1 deleted file mode 100644 index a86fbc35..00000000 --- a/tests/testruns/rfc5424-2.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. -165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. -#Example from RFC5424, section 6.5 / sample 2 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-3.parse1 b/tests/testruns/rfc5424-3.parse1 deleted file mode 100644 index 6ad4073d..00000000 --- a/tests/testruns/rfc5424-3.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"][examplePriority@32473 class="high"] -165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog, -#Example from RFC5424, section 6.5 / sample 4 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-4.parse1 b/tests/testruns/rfc5424-4.parse1 deleted file mode 100644 index ecf27e14..00000000 --- a/tests/testruns/rfc5424-4.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"] BOMAn application event log entry... -165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog,BOMAn application event log entry... -#Example from RFC5424, section 6.5 / sample 3 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.omod-if-array b/tests/testsuites/1.omod-if-array new file mode 100644 index 00000000..c464b19c --- /dev/null +++ b/tests/testsuites/1.omod-if-array @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, diff --git a/tests/testsuites/1.parse1 b/tests/testsuites/1.parse1 new file mode 100644 index 00000000..5ae655e6 --- /dev/null +++ b/tests/testsuites/1.parse1 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,local4,debug,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/omod-if-array.conf b/tests/testsuites/omod-if-array.conf new file mode 100644 index 00000000..e6c05a52 --- /dev/null +++ b/tests/testsuites/omod-if-array.conf @@ -0,0 +1,14 @@ +# Test config for array-passing output module interface +# (stanard string passing is already tested via the other test inside +# the testbench, so we do not need to focus on that) +# rgerhards, 2009-04-03 +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 12514 + +$ActionOMStdoutArrayInterface on +$ErrorMessagesToStderr off + +# do NOT remove \n, that would hang the test driver! +$template expect,"%PRI%%timestamp%%hostname%%programname%%syslogtag%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/parse1.conf b/tests/testsuites/parse1.conf new file mode 100644 index 00000000..0fb7d16d --- /dev/null +++ b/tests/testsuites/parse1.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 12514 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/rfc3164.parse1 b/tests/testsuites/rfc3164.parse1 new file mode 100644 index 00000000..e7a5fa18 --- /dev/null +++ b/tests/testsuites/rfc3164.parse1 @@ -0,0 +1,4 @@ +<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine,su,su:, 'su root' failed for lonvick on /dev/pts/8 +#Example from RFC3164, section 5.4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1 new file mode 100644 index 00000000..23836c9f --- /dev/null +++ b/tests/testsuites/rfc5424-1.parse1 @@ -0,0 +1,3 @@ +#Example from RFC5424, section 6.5 / sample 1 +<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 diff --git a/tests/testsuites/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1 new file mode 100644 index 00000000..a86fbc35 --- /dev/null +++ b/tests/testsuites/rfc5424-2.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. +165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. +#Example from RFC5424, section 6.5 / sample 2 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/rfc5424-3.parse1 b/tests/testsuites/rfc5424-3.parse1 new file mode 100644 index 00000000..6ad4073d --- /dev/null +++ b/tests/testsuites/rfc5424-3.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"][examplePriority@32473 class="high"] +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog, +#Example from RFC5424, section 6.5 / sample 4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/rfc5424-4.parse1 b/tests/testsuites/rfc5424-4.parse1 new file mode 100644 index 00000000..ecf27e14 --- /dev/null +++ b/tests/testsuites/rfc5424-4.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"] BOMAn application event log entry... +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog,BOMAn application event log entry... +#Example from RFC5424, section 6.5 / sample 3 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/udptester.c b/tests/udptester.c new file mode 100644 index 00000000..ea642db6 --- /dev/null +++ b/tests/udptester.c @@ -0,0 +1,291 @@ +/* Runs a test suite on the rsyslog (and later potentially + * other things). + * + * The name of the test suite must be given as argv[1]. In this config, + * rsyslogd is loaded with config ./testsuites/.conf and then + * test cases ./testsuites/ *. are executed on it. This test driver is + * suitable for testing cases where a message sent (via UDP) results in + * exactly one response. It can not be used in cases where no response + * is expected (that would result in a hang of the test driver). + * Note: each test suite can contain many tests, but they all need to work + * with the same rsyslog configuration. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 + +static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ +static char *testSuite; /* name of current test suite */ + + +void readLine(int fd, char *ln) +{ + char c; + int lenRead; + lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + *ln++ = c; + lenRead = read(fd, &c, 1); + } + *ln = '\0'; +} + + +/* send a message via UDP + * returns 0 if ok, something else otherwise. + */ +int +udpSend(char *buf, int lenBuf) +{ + struct sockaddr_in si_other; + int s, slen=sizeof(si_other); + + if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &si_other, 0, sizeof(si_other)); + si_other.sin_family = AF_INET; + si_other.sin_port = htons(12514); + if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + + if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { + perror("sendto"); + fprintf(stderr, "sendto() failed\n"); + return(1); + } + + close(s); + return 0; +} + + +/* open pipe to test candidate - so far, this is + * always rsyslogd and with a fixed config. Later, we may + * change this. Returns 0 if ok, something else otherwise. + * rgerhards, 2009-03-31 + */ +int openPipe(char *configFile, pid_t *pid, int *pfd) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", + "-M../runtime//.libs", NULL }; + char confFile[1024]; + char *newenviron[] = { NULL }; + + + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); + newargv[1] = confFile; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if(cpid == 0) { /* Child reads from pipe */ + fclose(stdout); + dup(pipefd[1]); + close(pipefd[1]); + close(pipefd[0]); + fclose(stdin); + execve("../tools/rsyslogd", newargv, newenviron); + } else { + close(pipefd[1]); + *pid = cpid; + *pfd = pipefd[0]; + } + + return(0); +} + + +/* Process a specific test case. File name is provided. + * Needs to return 0 if all is OK, something else otherwise. + */ +int +processTestFile(int fd, char *pszFileName) +{ + FILE *fp; + char *testdata = NULL; + char *expected = NULL; + int ret = 0; + size_t lenLn; + char buf[4096]; + + if((fp = fopen((char*)pszFileName, "r")) == NULL) { + perror((char*)pszFileName); + return(2); + } + + /* skip comments at start of file */ + + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + + + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send */ + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + free(testdata); + free(expected); + fclose(fp); + return(ret); +} + + +/* carry out all tests. Tests are specified via a file name + * wildcard. Each of the files is read and the test carried + * out. + * Returns the number of tests that failed. Zero means all + * success. + */ +int +doTests(int fd, char *files) +{ + int iFailed = 0; + int iTests = 0; + int ret; + char *testFile; + glob_t testFiles; + size_t i = 0; + struct stat fileInfo; + + glob(files, GLOB_MARK, NULL, &testFiles); + + for(i = 0; i < testFiles.gl_pathc; i++) { + testFile = testFiles.gl_pathv[i]; + + if(stat((char*) testFile, &fileInfo) != 0) + continue; /* continue with the next file if we can't stat() the file */ + + ++iTests; + /* all regular files are run through the test logic. Symlinks don't work. */ + if(S_ISREG(fileInfo.st_mode)) { /* config file */ + printf("processing test case '%s' ... ", testFile); + ret = processTestFile(fd, testFile); + if(ret == 0) { + printf("successfully completed\n"); + } else { + printf("failed!\n"); + ++iFailed; + } + } + } + globfree(&testFiles); + + if(iTests == 0) { + printf("Error: no test cases found, no tests executed.\n"); + iFailed = 1; + } else { + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + } + + return(iFailed); +} + + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int fd; + pid_t pid; + int ret = 0; + char buf[4096]; + char testcases[4096]; + + if(argc != 2) { + printf("Invalid call of udptester\n"); + printf("Usage: udptester testsuite-name\n"); + exit(1); + } + + testSuite = argv[1]; + + if((srcdir = getenv("srcdir")) == NULL) + srcdir = "."; + + printf("Start of udptester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + + openPipe(argv[1], &pid, &fd); + readLine(fd, buf); + + /* generate filename */ + sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); + if(doTests(fd, testcases) != 0) + ret = 1; + + /* cleanup */ + kill(pid, SIGTERM); + printf("End of udptester run.\n"); + exit(ret); +} -- cgit From 04b06af335bedcb651ccffd852863b7d17bf20dc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 18:20:52 +0200 Subject: improved parser test suite new tests added, now much better --- runtime/datetime.c | 2 +- tests/testsuites/2.parse1 | 3 +++ tests/testsuites/date1.parse1 | 3 +++ tests/testsuites/date2.parse1 | 3 +++ tests/testsuites/date3.parse1 | 3 +++ tests/testsuites/date4.parse1 | 3 +++ tests/testsuites/date5.parse1 | 3 +++ 7 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/testsuites/2.parse1 create mode 100644 tests/testsuites/date1.parse1 create mode 100644 tests/testsuites/date2.parse1 create mode 100644 tests/testsuites/date3.parse1 create mode 100644 tests/testsuites/date4.parse1 create mode 100644 tests/testsuites/date5.parse1 diff --git a/runtime/datetime.c b/runtime/datetime.c index 676f76d5..deb66eb4 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -479,7 +479,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) if(hour > 1970 && hour < 2100) { /* if so, we assume this actually is a year. This is a format found * e.g. in Cisco devices. - * (if you read this 2100+ trying to fix a bug, congratulate myself + * (if you read this 2100+ trying to fix a bug, congratulate me * to how long the code survived - me no longer ;)) -- rgerhards, 2008-11-18 */ year = hour; diff --git a/tests/testsuites/2.parse1 b/tests/testsuites/2.parse1 new file mode 100644 index 00000000..628e06df --- /dev/null +++ b/tests/testsuites/2.parse1 @@ -0,0 +1,3 @@ +<38>Mar 27 19:06:53 source_server sshd(pam_unix)[12750]: session opened for user foo by (uid=0) +38,auth,info,Mar 27 19:06:53,source_server,sshd(pam_unix),sshd(pam_unix)[12750]:, session opened for user foo by (uid=0) +# yet another real-life sample where we had some issues with diff --git a/tests/testsuites/date1.parse1 b/tests/testsuites/date1.parse1 new file mode 100644 index 00000000..ffc7c373 --- /dev/null +++ b/tests/testsuites/date1.parse1 @@ -0,0 +1,3 @@ +<38> Mar 7 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# one space in front of the date diff --git a/tests/testsuites/date2.parse1 b/tests/testsuites/date2.parse1 new file mode 100644 index 00000000..8d587d9d --- /dev/null +++ b/tests/testsuites/date2.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# only one space between "Mar" and "7" diff --git a/tests/testsuites/date3.parse1 b/tests/testsuites/date3.parse1 new file mode 100644 index 00000000..940d261e --- /dev/null +++ b/tests/testsuites/date3.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 2008 19:06:53: example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# the year should not be there, nor the colon after the date, but we accept it... diff --git a/tests/testsuites/date4.parse1 b/tests/testsuites/date4.parse1 new file mode 100644 index 00000000..eee5fb09 --- /dev/null +++ b/tests/testsuites/date4.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 2008 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# the year should not be there, but we accept it... diff --git a/tests/testsuites/date5.parse1 b/tests/testsuites/date5.parse1 new file mode 100644 index 00000000..be32e605 --- /dev/null +++ b/tests/testsuites/date5.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 19:06:53: example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# colon after timestamp is strictly not ok, but we accept it -- cgit From 010060289a729dd930ac04b72237f0ca0db9ea1d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 10:56:27 +0200 Subject: made sure udptester terminates only after rsyslgod it spawned We noticed this race issue under Solaris (thanks to its different scheduler, I guess). In some cases, the previous instance of rsyslogd was not terminated, resulting in a failure on the next test. Now handled correctly. --- tests/udptester.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/udptester.c b/tests/udptester.c index ea642db6..a3c6658d 100644 --- a/tests/udptester.c +++ b/tests/udptester.c @@ -259,6 +259,7 @@ int main(int argc, char *argv[]) { int fd; pid_t pid; + int status; int ret = 0; char buf[4096]; char testcases[4096]; @@ -286,6 +287,7 @@ int main(int argc, char *argv[]) /* cleanup */ kill(pid, SIGTERM); + waitpid(pid, &status, 0); /* wait until instance terminates */ printf("End of udptester run.\n"); exit(ret); } -- cgit From d30388a499ed134b7f0b64180d03d5c7959f2fe4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Apr 2009 15:17:49 +0200 Subject: updating changelog for release --- ChangeLog | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 694ee4b8..adf36932 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,13 +1,11 @@ --------------------------------------------------------------------------- -Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation of CSV-formatted outputs (format from RFC4180 is used) - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet supported - but we now have a good foundation to do that later on. - NOTE: nested function calls are not yet supported due to a design - issue with the function call VM instruction set design. - implemented the strlen() RainerScript function - added a template output module - added -T rsyslogd command line option, enables to specify a directory @@ -36,6 +34,10 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? more effcient for some outputs. They new can receive fields not only as a single string but rather in an array where each string is seperated. - added (some) developer documentation for output plugin interface +- bugfix: potential abort with DA queue after high watermark is reached + There exists a race condition that can lead to a segfault. Thanks + go to vbernetr, who performed the analysis and provided patch, which + I only tweaked a very little bit. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages -- cgit From 13344f4928654fc33cc37406164a4a56d22fff93 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Tue, 7 Apr 2009 11:15:52 +0200 Subject: Re-enable parsing host names from message. There was a subtle bug that made all messages fill their HOSTNAME from the source IP (which may be wrong in a long chain of relays) and not by reading the message. This fixes it. --- tools/syslogd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index b23c12a7..a40b34dd 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1496,7 +1496,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the fields. I think this logic shall work with any type of syslog message. */ bTAGCharDetected = 0; - if(pMsg->bParseHOSTNAME) { + if(flags & PARSE_HOSTNAME) { /* TODO: quick and dirty memory allocation */ /* the memory allocated is far too much in most cases. But on the plus side, * it is quite fast... - rgerhards, 2007-09-20 -- cgit From 845c6f59b91e9988f856556cbb0e88e275e8e591 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Apr 2009 15:23:12 +0200 Subject: final touches for 4.1.6 release --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index adf36932..937ee22f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which I only tweaked a very little bit. +- bugfix: imtcp did incorrectly parse hostname/tag + Thanks to Luis Fernando Muñoz Mejías for the patch. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages -- cgit From 5ccec36e8d69230ddbd94d1b63bc9f6518c5ade8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 14:58:46 +0200 Subject: highlighting $DirCreateMode fix --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 6f8c96ae..ac9d9fd6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,9 @@ Version 4.1.7 [BETA] (rgerhards), 2009-04-?? - bugfix: $InputTCPMaxSessions config directive was accepted, but not honored. This resulted in a fixed upper limit of 200 connections. +- bugfix: the default for $DirCreateMode was 0644, and as such wrong. + It has now been changed to 0700. For some background, please see + http://lists.adiscon.net/pipermail/rsyslog/2009-April/001986.html --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation -- cgit From 8e536c5b25ca1a7106f541149cf0d76bdf9237da Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:21:22 +0200 Subject: highlighted bugfix imported from beta --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index ccd1e7ae..b1d3b8d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,11 @@ Version 4.1.7 [BETA] (rgerhards), 2009-04-?? - bugfix: the default for $DirCreateMode was 0644, and as such wrong. It has now been changed to 0700. For some background, please see http://lists.adiscon.net/pipermail/rsyslog/2009-April/001986.html +- bugfix: ompgsql did not detect problems in sql command execution + this could cause loss of messages. The handling was correct if the + connection broke, but not if there was a problem with statement + execution. The most probable case for such a case would be invalid + sql inside the template, and this is now much easier to diagnose. --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation -- cgit