From d61eb8067b11bdee20e64c73345df2537275e03a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 21 Dec 2011 14:48:54 +0100 Subject: prepared milestone (version push) --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 99d7b203..a4bc7f88 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.8.6-newstats1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.8.6-newstats2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) -- cgit From 7774a3e8744f447f262dfbb834af626a94acced6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 8 Jan 2012 17:41:08 +0100 Subject: cleanup of imudp to use better lstn data structure (part I) --- plugins/imudp/imudp.c | 128 +++++++++++++++++++++++--------------------------- 1 file changed, 60 insertions(+), 68 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 32cd89d2..3fb60727 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -72,15 +72,20 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +static struct cnfinfo_s { +int udpLstnSocks; /* Internet datagram sockets, first element is nbr of elements + * read-only after init(), but beware of restart! */ +ruleset_t *udpRulesets; /* ruleset to be used with sockets in question (entry 0 is empty) */ +} *lcnfinfo = NULL; /**< the structure contains information needed to configure the listeners */ +static int nLstn = 0; /**< number of listners */ + static int bDoACLCheck; /* are ACL checks neeed? Cached once immediately before listener startup */ static int iMaxLine; /* maximum UDP message size supported */ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitted sender was last discarded * This shall prevent remote DoS when the "discard on disallowed sender" * message is configured to be logged on occurance of such a case. */ -static int *udpLstnSocks = NULL; /* Internet datagram sockets, first element is nbr of elements - * read-only after init(), but beware of restart! */ -static ruleset_t **udpRulesets = NULL; /* ruleset to be used with sockets in question (entry 0 is empty) */ + static uchar *pszBindAddr = NULL; /* IP to bind socket to */ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a global and alloc * it so that we can check available memory in willRun() and request @@ -195,9 +200,9 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) DEFiRet; uchar *bindAddr; int *newSocks; - int *tmpSocks; int iSrc, iDst; - ruleset_t **tmpRulesets; + struct cnfinfo_s *newlcnfinfo; + int newnLstn; /* check which address to bind to. We could do this more compact, but have not * done so in order to make the code more readable. -- rgerhards, 2007-12-27 @@ -215,48 +220,34 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) newSocks = net.create_udp_socket(bindAddr, (pNewVal == NULL || *pNewVal == '\0') ? (uchar*) "514" : pNewVal, 1); if(newSocks != NULL) { /* we now need to add the new sockets to the existing set */ - if(udpLstnSocks == NULL) { - /* esay, we can just replace it */ - udpLstnSocks = newSocks; - CHKmalloc(udpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (newSocks[0] + 1))); - for(iDst = 1 ; iDst <= newSocks[0] ; ++iDst) - udpRulesets[iDst] = pBindRuleset; + if(lcnfinfo == NULL) { + /* esay, src and dest are equal */ + nLstn = newSocks[0]; + CHKmalloc(lcnfinfo = (struct cnfinfo_s*) MALLOC(sizeof(struct cnfinfo_s) * nLstn)); + for(iDst = 0, iSrc=1 ; iDst < nLstn ; ++iDst, ++iSrc) { + lcnfinfo[iDst].udpLstnSocks = newSocks[iSrc]; + lcnfinfo[iDst].udpRulesets = pBindRuleset; + } } else { /* we need to add them */ - tmpSocks = (int*) MALLOC(sizeof(int) * (1 + newSocks[0] + udpLstnSocks[0])); - tmpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (1 + newSocks[0] + udpLstnSocks[0])); - if(tmpSocks == NULL || tmpRulesets == NULL) { - DBGPRINTF("out of memory trying to allocate udp listen socket array\n"); - /* in this case, we discard the new sockets but continue with what we - * already have - */ - free(newSocks); - free(tmpSocks); - free(tmpRulesets); - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } else { - /* ready to copy */ - iDst = 1; - for(iSrc = 1 ; iSrc <= udpLstnSocks[0] ; ++iSrc, ++iDst) { - tmpSocks[iDst] = udpLstnSocks[iSrc]; - tmpRulesets[iDst] = udpRulesets[iSrc]; - } - for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc, ++iDst) { - tmpSocks[iDst] = newSocks[iSrc]; - tmpRulesets[iDst] = pBindRuleset; - } - tmpSocks[0] = udpLstnSocks[0] + newSocks[0]; - free(newSocks); - free(udpLstnSocks); - udpLstnSocks = tmpSocks; - free(udpRulesets); - udpRulesets = tmpRulesets; + newnLstn = nLstn + newSocks[0]; + CHKmalloc(newlcnfinfo = (struct cnfinfo_s*) MALLOC(sizeof(struct cnfinfo_s) * newnLstn)); + /* ready to copy */ + iDst = 0; + memcpy(newlcnfinfo, lcnfinfo, nLstn * sizeof(struct cnfinfo_s)); + for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc, ++iDst) { + lcnfinfo[iDst].udpLstnSocks = newSocks[iSrc]; + lcnfinfo[iDst].udpRulesets = pBindRuleset; } + free(lcnfinfo); + lcnfinfo = newlcnfinfo; + nLstn = newnLstn; } } finalize_it: free(pNewVal); /* in any case, this is no longer needed */ + free(newSocks); RETiRet; } @@ -299,8 +290,7 @@ finalize_it: * on scheduling order. -- rgerhards, 2008-10-02 */ static inline rsRetVal -processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, - ruleset_t *pRuleset) +processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetPrev, int *pbIsPermitted) { DEFiRet; int iNbrTimeUsed; @@ -320,7 +310,7 @@ processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, if(pThrd->bShallStop == TRUE) ABORT_FINALIZE(RS_RET_FORCE_TERM); socklen = sizeof(struct sockaddr_storage); - lenRcvBuf = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); + lenRcvBuf = recvfrom(lcnfinfo[idxLstn].udpLstnSocks, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if(lenRcvBuf < 0) { if(errno != EINTR && errno != EAGAIN) { rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -365,7 +355,7 @@ processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", fd, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", lcnfinfo[idxLstn].udpLstnSocks, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); if(*pbIsPermitted != 0) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { @@ -375,7 +365,7 @@ processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); MsgSetInputName(pMsg, pInputName); - MsgSetRuleset(pMsg, pRuleset); + MsgSetRuleset(pMsg, lcnfinfo[idxLstn].udpRulesets); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; if(*pbIsPermitted == 2) @@ -436,7 +426,8 @@ static void set_thread_schedparam(void) * interface. ./configure settings control which one is used. * rgerhards, 2009-09-09 */ -#if defined(HAVE_EPOLL_CREATE1) || defined(HAVE_EPOLL_CREATE) +//#if defined(HAVE_EPOLL_CREATE1) || defined(HAVE_EPOLL_CREATE) +#if 0 #define NUM_EPOLL_EVENTS 10 rsRetVal rcvMainLoop(thrdInfo_t *pThrd) { @@ -457,7 +448,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); - CHKmalloc(udpEPollEvt = calloc(udpLstnSocks[0], sizeof(struct epoll_event))); + CHKmalloc(udpEPollEvt = calloc(nLstn, sizeof(struct epoll_event))); #if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) DBGPRINTF("imudp uses epoll_create1()\n"); @@ -477,14 +468,14 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) /* fill the epoll set - we need to do this only once, as the set * can not change dyamically. */ - for (i = 0; i < *udpLstnSocks; i++) { - if (udpLstnSocks[i+1] != -1) { + for (i = 0; i < nLstn ; i++) { + if (lcnfinfo[i].udpLstnSocks != -1) { udpEPollEvt[i].events = EPOLLIN | EPOLLET; - udpEPollEvt[i].data.u64 = i+1; - if(epoll_ctl(efd, EPOLL_CTL_ADD, udpLstnSocks[i+1], &(udpEPollEvt[i])) < 0) { + udpEPollEvt[i].data.u64 = i; + if(epoll_ctl(efd, EPOLL_CTL_ADD, lcnfinfo[i].udpLstnSocks, &(udpEPollEvt[i])) < 0) { rs_strerror_r(errno, errStr, sizeof(errStr)); errmsg.LogError(errno, NO_ERRCODE, "epoll_ctrl failed on fd %d with %s\n", - udpLstnSocks[i+1], errStr); + lcnfinfo[i].udpLstnSocks, errStr); } } } @@ -498,8 +489,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) break; /* terminate input! */ for(i = 0 ; i < nfds ; ++i) { - processSocket(pThrd, udpLstnSocks[currEvt[i].data.u64], &frominetPrev, &bIsPermitted, - udpRulesets[currEvt[i].data.u64]); + processSocket(pThrd, (int)currEvt[i].data.u64, &frominetPrev, &bIsPermitted); } } @@ -540,12 +530,12 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) FD_ZERO(&readfds); /* Add the UDP listen sockets to the list of read descriptors. */ - for (i = 0; i < *udpLstnSocks; i++) { - if (udpLstnSocks[i+1] != -1) { + for (i = 0; i < nLstn ; i++) { + if (lcnfinfo[i].udpLstnSocks != -1) { if(Debug) - net.debugListenInfo(udpLstnSocks[i+1], "UDP"); - FD_SET(udpLstnSocks[i+1], &readfds); - if(udpLstnSocks[i+1]>maxfds) maxfds=udpLstnSocks[i+1]; + net.debugListenInfo(lcnfinfo[i].udpLstnSocks, "UDP"); + FD_SET(lcnfinfo[i].udpLstnSocks, &readfds); + if(lcnfinfo[i].udpLstnSocks>maxfds) maxfds=lcnfinfo[i].udpLstnSocks; } } if(Debug) { @@ -561,10 +551,9 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ - for(i = 0; nfds && i < *udpLstnSocks; i++) { - if(FD_ISSET(udpLstnSocks[i+1], &readfds)) { - processSocket(pThrd, udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, - udpRulesets[i+1]); + for(i = 0; nfds && i < nLstn ; i++) { + if(FD_ISSET(lcnfinfo[i].udpLstnSocks, &readfds)) { + processSocket(pThrd, i, &frominetPrev, &bIsPermitted); --nfds; /* indicate we have processed one descriptor */ } } @@ -597,7 +586,7 @@ CODESTARTwillRun net.HasRestrictions(UCHAR_CONSTANT("UDP"), &bDoACLCheck); /* UDP */ /* if we could not set up any listners, there is no point in running... */ - if(udpLstnSocks == NULL) + if(nLstn == 0) ABORT_FINALIZE(RS_RET_NO_RUN); iMaxLine = glbl.GetMaxLine(); @@ -611,12 +600,15 @@ BEGINafterRun CODESTARTafterRun /* do cleanup here */ net.clearAllowedSenders((uchar*)"UDP"); - if(udpLstnSocks != NULL) { - net.closeUDPListenSockets(udpLstnSocks); - udpLstnSocks = NULL; - free(udpRulesets); - udpRulesets = NULL; +#warning UDP listen socks must be cloesed! also select must be supported! +#if 0 + if(lcnfinfo.udpLstnSocks != NULL) { + net.closeUDPListenSockets(lcnfinfo.udpLstnSocks); + lcnfinfo.udpLstnSocks = NULL; + free(lcnfinfo.udpRulesets); + lcnfinfo.udpRulesets = NULL; } +#endif if(pRcvBuf != NULL) { free(pRcvBuf); pRcvBuf = NULL; -- cgit From 6ea4cb7f5383f8c367be810fdc66650eeb9cc787 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 8 Jan 2012 17:42:55 +0100 Subject: nit fixed (testing aid removed) --- plugins/imudp/imudp.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 3fb60727..6a6ab498 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -426,8 +426,7 @@ static void set_thread_schedparam(void) * interface. ./configure settings control which one is used. * rgerhards, 2009-09-09 */ -//#if defined(HAVE_EPOLL_CREATE1) || defined(HAVE_EPOLL_CREATE) -#if 0 +#if defined(HAVE_EPOLL_CREATE1) || defined(HAVE_EPOLL_CREATE) #define NUM_EPOLL_EVENTS 10 rsRetVal rcvMainLoop(thrdInfo_t *pThrd) { -- cgit From 5438784b3ff566dc9efd863151ff782e2b9f4e45 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 Jan 2012 09:01:44 +0100 Subject: imudp refactor: fixed socket leak --- plugins/imudp/imudp.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 6a6ab498..f5a8a08d 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -596,18 +596,14 @@ ENDwillRun BEGINafterRun + int i; CODESTARTafterRun /* do cleanup here */ net.clearAllowedSenders((uchar*)"UDP"); -#warning UDP listen socks must be cloesed! also select must be supported! -#if 0 - if(lcnfinfo.udpLstnSocks != NULL) { - net.closeUDPListenSockets(lcnfinfo.udpLstnSocks); - lcnfinfo.udpLstnSocks = NULL; - free(lcnfinfo.udpRulesets); - lcnfinfo.udpRulesets = NULL; - } -#endif + for (i = 0; i < nLstn ; i++) + close(lcnfinfo[i].udpLstnSocks); + free(lcnfinfo); + lcnfinfo = NULL; if(pRcvBuf != NULL) { free(pRcvBuf); pRcvBuf = NULL; -- cgit From c1ac395b0b6010938827225739d1861c2b1fa63a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 Jan 2012 09:36:15 +0100 Subject: imudp refactor: regression fix --- plugins/imudp/imudp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index f5a8a08d..7068af0b 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -233,11 +233,11 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) newnLstn = nLstn + newSocks[0]; CHKmalloc(newlcnfinfo = (struct cnfinfo_s*) MALLOC(sizeof(struct cnfinfo_s) * newnLstn)); /* ready to copy */ - iDst = 0; + iDst = nLstn; memcpy(newlcnfinfo, lcnfinfo, nLstn * sizeof(struct cnfinfo_s)); for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc, ++iDst) { - lcnfinfo[iDst].udpLstnSocks = newSocks[iSrc]; - lcnfinfo[iDst].udpRulesets = pBindRuleset; + newlcnfinfo[iDst].udpLstnSocks = newSocks[iSrc]; + newlcnfinfo[iDst].udpRulesets = pBindRuleset; } free(lcnfinfo); lcnfinfo = newlcnfinfo; -- cgit From c0b3c93a8b16d4b7aaf912405fce77a4be365f0a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 Jan 2012 09:43:01 +0100 Subject: imudp refactor: field renaming (new names more more sense) --- plugins/imudp/imudp.c | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 7068af0b..0e6c6614 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -73,9 +73,8 @@ statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) static struct cnfinfo_s { -int udpLstnSocks; /* Internet datagram sockets, first element is nbr of elements - * read-only after init(), but beware of restart! */ -ruleset_t *udpRulesets; /* ruleset to be used with sockets in question (entry 0 is empty) */ + int sock; /* socket */ + ruleset_t *pRuleset; /* bound ruleset */ } *lcnfinfo = NULL; /**< the structure contains information needed to configure the listeners */ static int nLstn = 0; /**< number of listners */ @@ -225,8 +224,8 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) nLstn = newSocks[0]; CHKmalloc(lcnfinfo = (struct cnfinfo_s*) MALLOC(sizeof(struct cnfinfo_s) * nLstn)); for(iDst = 0, iSrc=1 ; iDst < nLstn ; ++iDst, ++iSrc) { - lcnfinfo[iDst].udpLstnSocks = newSocks[iSrc]; - lcnfinfo[iDst].udpRulesets = pBindRuleset; + lcnfinfo[iDst].sock = newSocks[iSrc]; + lcnfinfo[iDst].pRuleset = pBindRuleset; } } else { /* we need to add them */ @@ -236,8 +235,8 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) iDst = nLstn; memcpy(newlcnfinfo, lcnfinfo, nLstn * sizeof(struct cnfinfo_s)); for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc, ++iDst) { - newlcnfinfo[iDst].udpLstnSocks = newSocks[iSrc]; - newlcnfinfo[iDst].udpRulesets = pBindRuleset; + newlcnfinfo[iDst].sock = newSocks[iSrc]; + newlcnfinfo[iDst].pRuleset = pBindRuleset; } free(lcnfinfo); lcnfinfo = newlcnfinfo; @@ -310,7 +309,7 @@ processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetP if(pThrd->bShallStop == TRUE) ABORT_FINALIZE(RS_RET_FORCE_TERM); socklen = sizeof(struct sockaddr_storage); - lenRcvBuf = recvfrom(lcnfinfo[idxLstn].udpLstnSocks, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); + lenRcvBuf = recvfrom(lcnfinfo[idxLstn].sock, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if(lenRcvBuf < 0) { if(errno != EINTR && errno != EAGAIN) { rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -355,7 +354,7 @@ processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetP *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", lcnfinfo[idxLstn].udpLstnSocks, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", lcnfinfo[idxLstn].sock, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); if(*pbIsPermitted != 0) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { @@ -365,7 +364,7 @@ processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetP CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); MsgSetInputName(pMsg, pInputName); - MsgSetRuleset(pMsg, lcnfinfo[idxLstn].udpRulesets); + MsgSetRuleset(pMsg, lcnfinfo[idxLstn].pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; if(*pbIsPermitted == 2) @@ -468,13 +467,13 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) * can not change dyamically. */ for (i = 0; i < nLstn ; i++) { - if (lcnfinfo[i].udpLstnSocks != -1) { + if (lcnfinfo[i].sock != -1) { udpEPollEvt[i].events = EPOLLIN | EPOLLET; udpEPollEvt[i].data.u64 = i; - if(epoll_ctl(efd, EPOLL_CTL_ADD, lcnfinfo[i].udpLstnSocks, &(udpEPollEvt[i])) < 0) { + if(epoll_ctl(efd, EPOLL_CTL_ADD, lcnfinfo[i].sock, &(udpEPollEvt[i])) < 0) { rs_strerror_r(errno, errStr, sizeof(errStr)); errmsg.LogError(errno, NO_ERRCODE, "epoll_ctrl failed on fd %d with %s\n", - lcnfinfo[i].udpLstnSocks, errStr); + lcnfinfo[i].sock, errStr); } } } @@ -530,11 +529,11 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) /* Add the UDP listen sockets to the list of read descriptors. */ for (i = 0; i < nLstn ; i++) { - if (lcnfinfo[i].udpLstnSocks != -1) { + if (lcnfinfo[i].sock != -1) { if(Debug) - net.debugListenInfo(lcnfinfo[i].udpLstnSocks, "UDP"); - FD_SET(lcnfinfo[i].udpLstnSocks, &readfds); - if(lcnfinfo[i].udpLstnSocks>maxfds) maxfds=lcnfinfo[i].udpLstnSocks; + net.debugListenInfo(lcnfinfo[i].sock, "UDP"); + FD_SET(lcnfinfo[i].sock, &readfds); + if(lcnfinfo[i].sock>maxfds) maxfds=lcnfinfo[i].sock; } } if(Debug) { @@ -551,7 +550,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) break; /* terminate input! */ for(i = 0; nfds && i < nLstn ; i++) { - if(FD_ISSET(lcnfinfo[i].udpLstnSocks, &readfds)) { + if(FD_ISSET(lcnfinfo[i].sock, &readfds)) { processSocket(pThrd, i, &frominetPrev, &bIsPermitted); --nfds; /* indicate we have processed one descriptor */ } @@ -564,7 +563,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) #endif /* #if HAVE_EPOLL_CREATE1 */ /* This function is called to gather input. - * Note that udpLstnSocks must be non-NULL because otherwise we would not have + * Note that sock must be non-NULL because otherwise we would not have * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 */ BEGINrunInput @@ -601,7 +600,7 @@ CODESTARTafterRun /* do cleanup here */ net.clearAllowedSenders((uchar*)"UDP"); for (i = 0; i < nLstn ; i++) - close(lcnfinfo[i].udpLstnSocks); + close(lcnfinfo[i].sock); free(lcnfinfo); lcnfinfo = NULL; if(pRcvBuf != NULL) { -- cgit From 355b6818b8ff08cd7e0d72f790bcc98504924e48 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 Jan 2012 11:55:57 +0100 Subject: imudp: added per-listner stats settings & more refactoring --- plugins/imudp/imudp.c | 154 ++++++++++++++++++++++++++------------------------ 1 file changed, 80 insertions(+), 74 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 0e6c6614..0db6bf9a 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2011 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -26,6 +26,7 @@ * A copy of the GPL can be found in the file "COPYING" in this distribution. */ #include "config.h" +#include #include #include #include @@ -69,14 +70,14 @@ DEFobjCurrIf(prop) DEFobjCurrIf(ruleset) DEFobjCurrIf(statsobj) -statsobj_t *modStats; -STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) -static struct cnfinfo_s { +static struct lstn_s { + struct lstn_s *next; int sock; /* socket */ ruleset_t *pRuleset; /* bound ruleset */ -} *lcnfinfo = NULL; /**< the structure contains information needed to configure the listeners */ -static int nLstn = 0; /**< number of listners */ + statsobj_t *stats; /* listener stats */ + STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +} *lcnfRoot = NULL, *lcnfLast = NULL; static int bDoACLCheck; /* are ACL checks neeed? Cached once immediately before listener startup */ static int iMaxLine; /* maximum UDP message size supported */ @@ -199,9 +200,11 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) DEFiRet; uchar *bindAddr; int *newSocks; - int iSrc, iDst; - struct cnfinfo_s *newlcnfinfo; - int newnLstn; + int iSrc; + struct lstn_s *newlcnfinfo; + uchar *bindName; + uchar *port; + uchar statname[64]; /* check which address to bind to. We could do this more compact, but have not * done so in order to make the code more readable. -- rgerhards, 2007-12-27 @@ -212,35 +215,37 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) bindAddr = NULL; else bindAddr = pszBindAddr; + bindName = (bindAddr == NULL) ? (uchar*)"*" : bindAddr; - DBGPRINTF("Trying to open syslog UDP ports at %s:%s.\n", - (bindAddr == NULL) ? (uchar*)"*" : bindAddr, pNewVal); + DBGPRINTF("Trying to open syslog UDP ports at %s:%s.\n", bindName, pNewVal); - newSocks = net.create_udp_socket(bindAddr, (pNewVal == NULL || *pNewVal == '\0') ? (uchar*) "514" : pNewVal, 1); + port = (pNewVal == NULL || *pNewVal == '\0') ? (uchar*) "514" : pNewVal; + newSocks = net.create_udp_socket(bindAddr, port, 1); if(newSocks != NULL) { /* we now need to add the new sockets to the existing set */ - if(lcnfinfo == NULL) { - /* esay, src and dest are equal */ - nLstn = newSocks[0]; - CHKmalloc(lcnfinfo = (struct cnfinfo_s*) MALLOC(sizeof(struct cnfinfo_s) * nLstn)); - for(iDst = 0, iSrc=1 ; iDst < nLstn ; ++iDst, ++iSrc) { - lcnfinfo[iDst].sock = newSocks[iSrc]; - lcnfinfo[iDst].pRuleset = pBindRuleset; - } - } else { - /* we need to add them */ - newnLstn = nLstn + newSocks[0]; - CHKmalloc(newlcnfinfo = (struct cnfinfo_s*) MALLOC(sizeof(struct cnfinfo_s) * newnLstn)); - /* ready to copy */ - iDst = nLstn; - memcpy(newlcnfinfo, lcnfinfo, nLstn * sizeof(struct cnfinfo_s)); - for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc, ++iDst) { - newlcnfinfo[iDst].sock = newSocks[iSrc]; - newlcnfinfo[iDst].pRuleset = pBindRuleset; - } - free(lcnfinfo); - lcnfinfo = newlcnfinfo; - nLstn = newnLstn; + /* ready to copy */ + for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc) { + CHKmalloc(newlcnfinfo = (struct lstn_s*) MALLOC(sizeof(struct lstn_s))); + newlcnfinfo->next = NULL; + newlcnfinfo->sock = newSocks[iSrc]; + newlcnfinfo->pRuleset = pBindRuleset; + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&(newlcnfinfo->stats))); + snprintf((char*)statname, sizeof(statname), "imudp(%s:%s)", bindName, port); + statname[sizeof(statname)-1] = '\0'; /* just to be on the save side... */ + CHKiRet(statsobj.SetName(newlcnfinfo->stats, statname)); + CHKiRet(statsobj.AddCounter(newlcnfinfo->stats, UCHAR_CONSTANT("submitted"), + ctrType_IntCtr, &(newlcnfinfo->ctrSubmit))); + CHKiRet(statsobj.ConstructFinalize(newlcnfinfo->stats)); + /* link to list. Order must be preserved to take care for + * conflicting matches. + */ + if(lcnfRoot == NULL) + lcnfRoot = newlcnfinfo; + if(lcnfLast == NULL) + lcnfLast = newlcnfinfo; + else + lcnfLast->next = newlcnfinfo; } } @@ -289,7 +294,7 @@ finalize_it: * on scheduling order. -- rgerhards, 2008-10-02 */ static inline rsRetVal -processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetPrev, int *pbIsPermitted) +processSocket(thrdInfo_t *pThrd, struct lstn_s *lstn, struct sockaddr_storage *frominetPrev, int *pbIsPermitted) { DEFiRet; int iNbrTimeUsed; @@ -309,7 +314,7 @@ processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetP if(pThrd->bShallStop == TRUE) ABORT_FINALIZE(RS_RET_FORCE_TERM); socklen = sizeof(struct sockaddr_storage); - lenRcvBuf = recvfrom(lcnfinfo[idxLstn].sock, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); + lenRcvBuf = recvfrom(lstn->sock, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if(lenRcvBuf < 0) { if(errno != EINTR && errno != EAGAIN) { rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -354,7 +359,7 @@ processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetP *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", lcnfinfo[idxLstn].sock, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", lstn->sock, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); if(*pbIsPermitted != 0) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { @@ -364,14 +369,14 @@ processSocket(thrdInfo_t *pThrd, int idxLstn, struct sockaddr_storage *frominetP CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); MsgSetInputName(pMsg, pInputName); - MsgSetRuleset(pMsg, lcnfinfo[idxLstn].pRuleset); + MsgSetRuleset(pMsg, lstn->pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; if(*pbIsPermitted == 2) pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */ CHKiRet(msgSetFromSockinfo(pMsg, &frominet)); CHKiRet(submitMsg(pMsg)); - STATSCOUNTER_INC(ctrSubmit, mutCtrSubmit); + STATSCOUNTER_INC(lstn->ctrSubmit, lstn->mutCtrSubmit); } } @@ -438,6 +443,8 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) struct epoll_event *udpEPollEvt = NULL; struct epoll_event currEvt[NUM_EPOLL_EVENTS]; char errStr[1024]; + struct lstn_s *lstn; + int nLstn; /* start "name caching" algo by making sure the previous system indicator * is invalidated. @@ -446,6 +453,10 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); + /* count num listeners -- do it here in order to avoid inconsistency */ + nLstn = 0; + for(lstn = lcnfRoot ; lstn != NULL ; lstn = lstn->next) + ++nLstn; CHKmalloc(udpEPollEvt = calloc(nLstn, sizeof(struct epoll_event))); #if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) @@ -466,16 +477,18 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) /* fill the epoll set - we need to do this only once, as the set * can not change dyamically. */ - for (i = 0; i < nLstn ; i++) { - if (lcnfinfo[i].sock != -1) { + i = 0; + for(lstn = lcnfRoot ; lstn != NULL ; lstn = lstn->next) { + if(lstn->sock != -1) { udpEPollEvt[i].events = EPOLLIN | EPOLLET; - udpEPollEvt[i].data.u64 = i; - if(epoll_ctl(efd, EPOLL_CTL_ADD, lcnfinfo[i].sock, &(udpEPollEvt[i])) < 0) { + udpEPollEvt[i].data.u64 = (long long unsigned) lstn; + if(epoll_ctl(efd, EPOLL_CTL_ADD, lstn->sock, &(udpEPollEvt[i])) < 0) { rs_strerror_r(errno, errStr, sizeof(errStr)); errmsg.LogError(errno, NO_ERRCODE, "epoll_ctrl failed on fd %d with %s\n", - lcnfinfo[i].sock, errStr); + lstn->sock, errStr); } } + i++; } while(1) { @@ -487,7 +500,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) break; /* terminate input! */ for(i = 0 ; i < nfds ; ++i) { - processSocket(pThrd, (int)currEvt[i].data.u64, &frominetPrev, &bIsPermitted); + processSocket(pThrd, (struct lstn_s*)currEvt[i].data.u64, &frominetPrev, &bIsPermitted); } } @@ -504,10 +517,10 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) DEFiRet; int maxfds; int nfds; - int i; fd_set readfds; struct sockaddr_storage frominetPrev; int bIsPermitted; + struct lstn_s *lstn; /* start "name caching" algo by making sure the previous system indicator * is invalidated. @@ -518,22 +531,18 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) DBGPRINTF("imudp uses select()\n"); while(1) { - /* Add the Unix Domain Sockets to the list of read - * descriptors. - * rgerhards 2005-08-01: we must now check if there are - * any local sockets to listen to at all. If the -o option - * is given without -a, we do not need to listen at all.. + /* Add the Unix Domain Sockets to the list of read descriptors. */ maxfds = 0; FD_ZERO(&readfds); /* Add the UDP listen sockets to the list of read descriptors. */ - for (i = 0; i < nLstn ; i++) { - if (lcnfinfo[i].sock != -1) { + for(lstn = lcnfRoot ; lstn != NULL ; lstn = lstn->next) { + if (lstn->sock != -1) { if(Debug) - net.debugListenInfo(lcnfinfo[i].sock, "UDP"); - FD_SET(lcnfinfo[i].sock, &readfds); - if(lcnfinfo[i].sock>maxfds) maxfds=lcnfinfo[i].sock; + net.debugListenInfo(lstn->sock, "UDP"); + FD_SET(lstn->sock, &readfds); + if(lstn->sock>maxfds) maxfds=lstn->sock; } } if(Debug) { @@ -549,9 +558,9 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ - for(i = 0; nfds && i < nLstn ; i++) { - if(FD_ISSET(lcnfinfo[i].sock, &readfds)) { - processSocket(pThrd, i, &frominetPrev, &bIsPermitted); + for(lstn = lcnfRoot ; nfds && lstn != NULL ; lstn = lstn->next) { + if(FD_ISSET(lstn->sock, &readfds)) { + processSocket(pThrd, lstn, &frominetPrev, &bIsPermitted); --nfds; /* indicate we have processed one descriptor */ } } @@ -584,8 +593,10 @@ CODESTARTwillRun net.HasRestrictions(UCHAR_CONSTANT("UDP"), &bDoACLCheck); /* UDP */ /* if we could not set up any listners, there is no point in running... */ - if(nLstn == 0) + if(lcnfRoot == NULL) { + DBGPRINTF("imudp: no listeners configured, will not run\n"); ABORT_FINALIZE(RS_RET_NO_RUN); + } iMaxLine = glbl.GetMaxLine(); @@ -595,14 +606,18 @@ ENDwillRun BEGINafterRun - int i; + struct lstn_s *lstn, *lstnDel; CODESTARTafterRun /* do cleanup here */ net.clearAllowedSenders((uchar*)"UDP"); - for (i = 0; i < nLstn ; i++) - close(lcnfinfo[i].sock); - free(lcnfinfo); - lcnfinfo = NULL; + for(lstn = lcnfRoot ; lstn != NULL ; ) { + statsobj.Destruct(&(lstn->stats)); + close(lstn->sock); + lstnDel = lstn; + lstn = lstn->next; + free(lstnDel); + } + lcnfRoot = lcnfLast = NULL; if(pRcvBuf != NULL) { free(pRcvBuf); pRcvBuf = NULL; @@ -614,8 +629,6 @@ ENDafterRun BEGINmodExit CODESTARTmodExit - statsobj.Destruct(&modStats); - /* release what we no longer need */ objRelease(errmsg, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); @@ -678,13 +691,6 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); - - /* support statistics gathering */ - CHKiRet(statsobj.Construct(&modStats)); - CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imudp"))); - CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), - ctrType_IntCtr, &ctrSubmit)); - CHKiRet(statsobj.ConstructFinalize(modStats)); ENDmodInit /* vim:set ai: */ -- cgit From 785e107502f58808152b3134915fd3c986b1ed27 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 Jan 2012 12:39:02 +0100 Subject: push version number --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a4bc7f88..dd305221 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.8.6-newstats2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.8.6-newstats3],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) -- cgit From 6766b6f3e7cf0331eea40403bae8145ac1fd805c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 Jan 2012 13:47:54 +0100 Subject: Merge v5-stable-newstats & manual ChangeLog update --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 641d6bea..ad6de170 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 5.9.5 [V5-DEVEL], 2011-11-29 +- new stats counters for imudp and imtcp - new stats counters "discarded.nf" and "discarded.full" for queue object. Tells how many messages have been discarded due to queue full condition. --------------------------------------------------------------------------- -- cgit From e25dc05e9127537cf2ddbae30b15300f601fbff0 Mon Sep 17 00:00:00 2001 From: Tomas Heinrich Date: Tue, 10 Jan 2012 17:07:55 +0100 Subject: added canned template for ultra-exact sysklogd file format the point is that LF will be translated to SP Signed-off-by: Rainer Gerhards --- doc/rsyslog_conf_templates.html | 4 ++++ tools/syslogd.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index 6c68b801..5790ba8a 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -118,6 +118,10 @@ with high-precision timestamps and timezone information useful if you send messages to other syslogd's or rsyslogd below version 3.12.5. +
  • RSYSLOG_SysklogdFileFormat +- sysklogd compatible log file format. If used with options: $SpaceLFOnReceive on; +$EscapeControlCharactersOnReceive off; $DropTrailingLFOnReception off, +the log format will conform to sysklogd log format.
  • RSYSLOG_ForwardFormat - a new high-precision forwarding format very similar to the traditional one, but with high-precision timestamps and timezone diff --git a/tools/syslogd.c b/tools/syslogd.c index 64b23566..4a5cbf60 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -381,6 +381,7 @@ static uchar template_TraditionalForwardFormat[] = "\"<%PRI%>%TIMESTAMP% %HOSTNA static uchar template_StdUsrMsgFmt[] = "\" %syslogtag%%msg%\n\r\""; static uchar template_StdDBFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-mysql%', '%timegenerated:::date-mysql%', %iut%, '%syslogtag%')\",SQL"; static uchar template_StdPgSQLFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%')\",STDSQL"; +static uchar template_SysklogdFileFormat[] = "\"%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%\n\""; /* end template */ @@ -2836,6 +2837,8 @@ static rsRetVal mainThread() tplAddLine(" StdUsrMsgFmt", &pTmp); pTmp = template_StdDBFmt; tplAddLine(" StdDBFmt", &pTmp); + pTmp = template_SysklogdFileFormat; + tplAddLine("RSYSLOG_SysklogdFileFormat", &pTmp); pTmp = template_StdPgSQLFmt; tplLastStaticInit(tplAddLine(" StdPgSQLFmt", &pTmp)); -- cgit From a2e69cffff21ad433e61b45f91a2be4d5ebc5c72 Mon Sep 17 00:00:00 2001 From: Tomas Heinrich Date: Tue, 10 Jan 2012 17:14:16 +0100 Subject: added $SpaceLFOnReceive config directive Signed-off-by: Rainer Gerhards --- dirty.h | 1 + doc/rsyslog_conf_global.html | 1 + runtime/parser.c | 6 +++++- tools/syslogd.c | 9 +++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dirty.h b/dirty.h index 0153cb69..a93eca8f 100644 --- a/dirty.h +++ b/dirty.h @@ -54,6 +54,7 @@ extern int bReduceRepeatMsgs; extern int bDropTrailingLF; extern uchar cCCEscapeChar; extern int bEscapeCCOnRcv; +extern int bSpaceLFOnRcv; #ifdef USE_NETZIP /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d5a27541..f5269490 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -130,6 +130,7 @@ our paper on using multiple rule sets in rsyslog$DropTrailingLFOnReception
  • $DynaFileCacheSize
  • $EscapeControlCharactersOnReceive
    • +
  • $SpaceLFOnReceive [on/off] - instructs rsyslogd to replace LF with spaces during message reception (sysklogd compatibility aid)
  • $ErrorMessagesToStderr [on|off] - direct rsyslogd error message to stderr (in addition to other targets)
  • $FailOnChownFailure
  • $FileCreateMode
    • diff --git a/runtime/parser.c b/runtime/parser.c index 466066e7..57b7bf8f 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -202,9 +202,13 @@ sanitizeMessage(msg_t *pMsg) int bNeedSanitize = 0; for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { if(iscntrl(pszMsg[iSrc])) { + if(bSpaceLFOnRcv && pszMsg[iSrc] == '\n') + pszMsg[iSrc] = ' '; + else if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { bNeedSanitize = 1; - break; + if (!bSpaceLFOnRcv) + break; } } } diff --git a/tools/syslogd.c b/tools/syslogd.c index 4a5cbf60..b696c55f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -259,6 +259,7 @@ static int bDebugPrintCfSysLineHandlerList = 1;/* output cfsyslinehandler list i static int bDebugPrintModuleList = 1;/* output module list in debug mode? */ uchar cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */ int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +int bSpaceLFOnRcv = 0; /* replace newlines with spaces on reception: 0 - no, 1 - yes */ static int bErrMsgToStderr = 1; /* print error messages to stderr (in addition to everything else)? */ int bReduceRepeatMsgs; /* reduce repeated message - 0 - no, 1 - yes */ int bActExecWhenPrevSusp; /* execute action only when previous one was suspended? */ @@ -343,6 +344,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bDebugPrintCfSysLineHandlerList = 1; bDebugPrintModuleList = 1; bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bSpaceLFOnRcv = 0; bReduceRepeatMsgs = 0; free(pszMainMsgQFName); pszMainMsgQFName = NULL; @@ -810,6 +812,9 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla /* log an error? Very questionable... rgerhards, 2006-11-30 */ /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ ++pData; + } else if (bSpaceLFOnRcv && *pData == '\n') { + *(pMsg + iMsg++) = ' '; + ++pData; } else if(bEscapeCCOnRcv && iscntrl((int) *pData)) { /* we are configured to escape control characters. Please note * that this most probably break non-western character sets like @@ -2115,6 +2120,9 @@ static void dbgPrintInitInfo(void) DBGPRINTF("Control characters are %sreplaced upon reception.\n", bEscapeCCOnRcv? "" : "not "); + DBGPRINTF("Newlines are %sreplaced upon reception.\n", + bSpaceLFOnRcv? "" : "not "); + if(bEscapeCCOnRcv) DBGPRINTF("Control character escape sequence prefix is '%c'.\n", cCCEscapeChar); @@ -2728,6 +2736,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeinterval", 0, eCmdHdlrInt, setActionResumeInterval, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"spacelfonreceive", 0, eCmdHdlrBinary, NULL, &bSpaceLFOnRcv, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"template", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_TEMPLATE, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"outchannel", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_OUTCHANNEL, NULL)); -- cgit From d4024e98d0a7cef5830c71992513bbef0e342874 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 10 Jan 2012 18:04:18 +0100 Subject: part of merge, but forgot to add to merge set --- runtime/parser.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime/parser.c b/runtime/parser.c index ffaaac25..300db1e0 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -60,6 +60,7 @@ DEFobjCurrIf(ruleset) /* config data */ static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bSpaceLFOnRcv = 0; /* replace newlines with spaces on reception: 0 - no, 1 - yes */ static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */ static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ @@ -649,6 +650,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus { cCCEscapeChar = '#'; bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bSpaceLFOnRcv = 0; bEscape8BitChars = 0; /* default is to escape control characters */ bEscapeTab = 1; /* default is to escape control characters */ bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ @@ -702,6 +704,7 @@ BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"spacelfonreceive", 0, eCmdHdlrBinary, NULL, &bSpaceLFOnRcv, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); -- cgit From 6b72037d669d7aa6672cb23d5190606ed7c3dd8d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 12 Jan 2012 04:05:34 +0100 Subject: regression fix: imuxsock-received messages were malformed probably caused by a recent merge, was not present in 5.9.4 --- plugins/imuxsock/imuxsock.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 0fb3492d..14d8b594 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -713,7 +713,6 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim lenRcv = toffs + 1; } - /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); @@ -734,8 +733,6 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim * datestamp or not .. and advance the parse pointer accordingly. */ datetime.ParseTIMESTAMP3164(&dummyTS, &parse, &lenMsg); - parse += 16; /* just skip timestamp */ - lenMsg -= 16; } else { if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { DBGPRINTF("we have a problem, invalid timestamp in msg!\n"); @@ -1018,7 +1015,6 @@ CODESTARTafterRun /* Clean-up files. */ for(i = startIndexUxLocalSockets; i < nfd; i++) if (listeners[i].sockName && listeners[i].fd != -1) { - /* If systemd passed us a socket it is systemd's job to clean it up. * Do not unlink it -- we will get same socket (node) from systemd * e.g. on restart again. -- cgit From 45ba66d14acd1910869c8e5c3fb1255150eeb629 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 14 Jan 2012 18:13:26 +0100 Subject: omfwd refactor: simplifying code also remove code introduced by commit 6e97513eea1a6e282365eb01d972e0657cb36baa --- tools/omfwd.c | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/tools/omfwd.c b/tools/omfwd.c index 10cce0e2..0156fbfc 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -131,22 +131,6 @@ pData->bIsConnected = 0; // TODO: remove this variable altogether } -/* get the syslog forward port from selector_t. The passed in - * struct must be one that is setup for forwarding. - * rgerhards, 2007-06-28 - * We may change the implementation to try to lookup the port - * if it is unspecified. So far, we use the IANA default auf 514. - */ -static char *getFwdPt(instanceData *pData) -{ - assert(pData != NULL); - if(pData->port == NULL) - return("514"); - else - return(pData->port); -} - - /* destruct the TCP helper objects * This, for example, is needed after something went wrong. * This function is void because it "can not" fail. @@ -345,7 +329,7 @@ static rsRetVal TCPSendInit(void *pvData) } /* params set, now connect */ CHKiRet(netstrm.Connect(pData->pNetstrm, glbl.GetDefPFFamily(), - (uchar*)getFwdPt(pData), (uchar*)pData->f_hname)); + (uchar*)pData->port, (uchar*)pData->f_hname)); } finalize_it: @@ -378,9 +362,9 @@ static rsRetVal doTryResume(instanceData *pData) hints.ai_flags = AI_NUMERICSERV; hints.ai_family = glbl.GetDefPFFamily(); hints.ai_socktype = SOCK_DGRAM; - if((iErr = (getaddrinfo(pData->f_hname, getFwdPt(pData), &hints, &res))) != 0) { + if((iErr = (getaddrinfo(pData->f_hname, pData->port, &hints, &res))) != 0) { dbgprintf("could not get addrinfo for hostname '%s':'%s': %d%s\n", - pData->f_hname, getFwdPt(pData), iErr, gai_strerror(iErr)); + pData->f_hname, pData->port, iErr, gai_strerror(iErr)); ABORT_FINALIZE(RS_RET_SUSPENDED); } dbgprintf("%s found, resuming.\n", pData->f_hname); @@ -420,7 +404,7 @@ CODESTARTdoAction iMaxLine = glbl.GetMaxLine(); - dbgprintf(" %s:%s/%s\n", pData->f_hname, getFwdPt(pData), + dbgprintf(" %s:%s/%s\n", pData->f_hname, pData->port, pData->protocol == FORW_UDP ? "udp" : "tcp"); psz = (char*) ppString[0]; @@ -630,12 +614,16 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store syslog forwarding port, " "using default port, results may not be what you intend\n"); - /* we leave f_forw.port set to NULL, this is then handled by getFwdPt(). */ + /* we leave f_forw.port set to NULL, this is then handled below */ } else { memcpy(pData->port, tmp, i); *(pData->port + i) = '\0'; } } + /* check if no port is set. If so, we use the IANA-assigned port of 514 */ + if(pData->port == NULL) { + CHKmalloc(pData->port = strdup("514")); + } /* now skip to template */ while(*p && *p != ';' && *p != '#' && !isspace((int) *p)) -- cgit From ac64c118ea9ea69edfc1ab698d5413fa3485fc94 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 14 Jan 2012 18:22:16 +0100 Subject: omfwd refactor: simplification at the same time, remove commits - 8c3ab2e26f1bae46ff34fc1d0a10a69c4db78127 - 009738a0ac6ba0dccf403f9e396095f44e4f9ac6 --- tools/omfwd.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tools/omfwd.c b/tools/omfwd.c index 0156fbfc..46c13dc9 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -396,9 +396,12 @@ CODESTARTtryResume ENDtryResume BEGINdoAction - char *psz = NULL; /* temporary buffering */ + char *psz; /* temporary buffering */ register unsigned l; int iMaxLine; +# ifdef USE_NETZIP + Bytef *out = NULL; /* for compression */ +# endif CODESTARTdoAction CHKiRet(doTryResume(pData)); @@ -422,7 +425,6 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { - Bytef *out; uLongf destLen = iMaxLine + iMaxLine/100 +12; /* recommended value from zlib doc */ uLong srcLen = l; int ret; @@ -443,14 +445,11 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ dbgprintf("Compression failed, sending uncompressed message\n"); - free(out); } else if(destLen+1 < l) { /* only use compression if there is a gain in using it! */ dbgprintf("there is gain in compression, so we do it\n"); psz = (char*) out; l = destLen + 1; /* take care for the "z" at message start! */ - } else { - free(out); } ++destLen; } @@ -472,10 +471,8 @@ CODESTARTdoAction } finalize_it: # ifdef USE_NETZIP - if((psz != NULL) && (psz != (char*) ppString[0])) { - /* we need to free temporary buffer, alloced above - Naoya Nakazawa, 2010-01-11 */ - free(psz); - } + if(out != NULL) + free(out); # endif ENDdoAction -- cgit From 38afe62f33db10a31ed2e4a6c6c10bc3995d860d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 14 Jan 2012 18:25:23 +0100 Subject: omfwd license change to ASL 2.0 after careful consideration of contributor history. David Lang gave permission to license under ASL 2.0, other patches have been replaced by refactored code. --- tools/omfwd.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/tools/omfwd.c b/tools/omfwd.c index 46c13dc9..acc2f6f6 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -4,30 +4,26 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c) - * This file is under development and has not yet arrived at being fully - * self-contained and a real object. So far, it is mostly an excerpt - * of the "old" message code without any modifications. However, it - * helps to have things at the right place one we go to the meat of it. + * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c, which + * at the time of rsyslog fork from sysklogd was under BSD license) * - * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2012 Adiscon GmbH. * * This file is part of rsyslog. * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "config.h" #include "rsyslog.h" -- cgit From a468063315c3ce5eccc702080bebeb79158ad6b7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 14 Jan 2012 18:27:24 +0100 Subject: cosmetic: cleanup of ChangeLog --- ChangeLog | 2 -- 1 file changed, 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4cf7dc19..ef742100 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,8 +3,6 @@ Version 5.9.5 [V5-DEVEL], 2011-11-29 - new stats counters for imudp and imtcp - new stats counters "discarded.nf" and "discarded.full" for queue object. Tells how many messages have been discarded due to queue full condition. ---------------------------------------------------------------------------- -Version 5.9.5 [V5-DEVEL], 2011-11-29 - enhanced module loader to not rely on PATH_MAX --------------------------------------------------------------------------- Version 5.9.4 [V5-DEVEL], 2011-11-29 -- cgit From d5a18a93ae585c25d67ec200b308f08c6bd6d5b2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 14 Jan 2012 18:29:33 +0100 Subject: omfwd refactor: even more simplification (small one ;)) --- tools/omfwd.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/omfwd.c b/tools/omfwd.c index acc2f6f6..b456db17 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -467,8 +467,7 @@ CODESTARTdoAction } finalize_it: # ifdef USE_NETZIP - if(out != NULL) - free(out); + free(out); /* is NULL if it was never used... */ # endif ENDdoAction -- cgit From 167192666ba8905b83210ab7c5f00cc6be9f7147 Mon Sep 17 00:00:00 2001 From: Peng Haitao Date: Fri, 6 Jan 2012 14:48:18 +0800 Subject: fix error of not define HAVE_SETSID [add list to the CC list] When HAVE_SETSID is not defined, rsyslogd will use ioctl() make itself to daemon, but this can not make rsyslogd process become the process group leader of a new process group. In RHEL6.1, the status is as follows: # uname -a Linux RHEL6U1GA-Intel64-199 2.6.32-131.0.15.el6.x86_64 #1 SMP Tue May 10 15:42:40 EDT 2011 x86_64 x86_64 x86_64 GNU/Linux # /etc/init.d/rsyslog restart Shutting down system logger: [ OK ] Starting system logger: [ OK ] # ps axo pgrp,ppid,pid,comm | grep rsyslog 6290 1 6301 rsyslogd When we send SIGTERM signal to 6290, rsyslogd will die:( So I think we should call setpgid() before ioctl(). Signed-off-by: Peng Haitao --- tools/syslogd.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/syslogd.c b/tools/syslogd.c index 6879bafa..65770404 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -460,8 +460,15 @@ void untty(void) #else { int i; + pid_t pid; if(!Debug) { + pid = getpid(); + if (setpgid(pid, pid) < 0) { + perror("setpgid"); + exit(1); + } + i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) -- cgit From 76c4d6b951453078ab53bf612caf0b8ec9d54bb8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Jan 2012 17:58:57 +0100 Subject: $IMUXSockRateLimitInterval DEFAULT CHANGED, was 5, now 0 The new default turns off rate limiting. This was chosen as people experienced problems with rate-limiting activated by default. Now it needs an explicit opt-in by setting this parameter. Thanks to Chris Gaffney for suggesting to make it opt-in; thanks to many unnamed others who already had complained at the time Chris made the suggestion ;-) --- ChangeLog | 9 +++++++++ doc/imuxsock.html | 5 ++++- plugins/imuxsock/imuxsock.c | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7639ca95..ad3f85f2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,13 @@ --------------------------------------------------------------------------- +Version 5.9.6 [V5-DEVEL], 2012-??-?? +- $IMUXSockRateLimitInterval DEFAULT CHANGED, was 5, now 0 + The new default turns off rate limiting. This was chosen as people + experienced problems with rate-limiting activated by default. Now it + needs an explicit opt-in by setting this parameter. + Thanks to Chris Gaffney for suggesting to make it opt-in; thanks to + many unnamed others who already had complained at the time Chris made + the suggestion ;-) +--------------------------------------------------------------------------- Version 5.9.5 [V5-DEVEL], 2011-11-29 - new stats counters for imudp and imtcp - new stats counters "discarded.nf" and "discarded.full" for queue object. diff --git a/doc/imuxsock.html b/doc/imuxsock.html index f80bc598..734ae889 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -65,7 +65,10 @@ you must turn it on (via $SystemLogSocketAnnotate and $InputUnixListenSocketAnno
  • $InputUnixListenSocketFlowControl [on/off] - specifies if flow control should be applied to the next socket.
  • $IMUXSockRateLimitInterval [number] - specifies the rate-limiting -interval in seconds. Default value is 5 seconds. Set it to 0 to turn rate limiting off. +interval in seconds. Default value is 0, which turns off rate limiting. Set it to a number +of seconds (5 recommended) to activate rate-limiting. The default of 0 has been choosen in 5.9.6+, +as people experienced problems with this feature activated by default. Now it needs an +explicit opt-in by setting this parameter.
  • $IMUXSockRateLimitBurst [number] - specifies the rate-limiting burst in number of messages. Default is 200. diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 14d8b594..403173e1 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -165,7 +165,7 @@ static int bAnnotate = 0; /* annotate trusted properties */ static int bAnnotateSysSock = 0; /* same, for system log socket */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ -#define DFLT_ratelimitInterval 5 +#define DFLT_ratelimitInterval 0 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 -- cgit From 052b1c75945106c6336fd0c172fbaa129248334c Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Mon, 15 Aug 2011 22:22:05 +1000 Subject: add JSON escaping option Following the path taken by the two SQL formatting options, which escape single quotes with double quotes (amongst other things), this patch adds a JSON quoting option. JSON is the opposite to the SQL options, requiring double quotes to be quoted within a string. This patch provides a formatting option implementing this requirement, while piggy-backing on the existing code as much as possible. Signed-off-by: Nathan Scott --- action.c | 2 +- template.c | 90 +++++++++++++++++++++++++++++++++++++++----------------------- template.h | 9 ++++--- 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/action.c b/action.c index 278625ce..6796cf14 100644 --- a/action.c +++ b/action.c @@ -1743,7 +1743,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques } /* check required template options */ if( (iTplOpts & OMSR_RQD_TPL_OPT_SQL) - && (pAction->ppTpl[i]->optFormatForSQL == 0)) { + && (pAction->ppTpl[i]->optFormatEscape == 0)) { errno = 0; errmsg.LogError(0, RS_RET_RQD_TPLOPT_MISSING, "Action disabled. To use this action, you have to specify " "the SQL or stdSQL option in your template!\n"); diff --git a/template.c b/template.c index 2038c6c1..54ca95af 100644 --- a/template.c +++ b/template.c @@ -53,7 +53,12 @@ static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ - +enum { + NO_ESCAPE = 0, + SQL_ESCAPE, + STDSQL_ESCAPE, + JSON_ESCAPE, +}; /* helper to tplToString and strgen's, extends buffer */ #define ALLOC_INC 128 @@ -123,10 +128,12 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * * but they are handled in this way because of legacy (don't break any * existing thing). */ - if(pTpl->optFormatForSQL == 1) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); - else if(pTpl->optFormatForSQL == 2) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); + if(pTpl->optFormatEscape == SQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, SQL_ESCAPE); + else if(pTpl->optFormatEscape == JSON_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, JSON_ESCAPE); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, STDSQL_ESCAPE); } /* got source, now copy over */ if(iLenVal > 0) { /* may be zero depending on property */ @@ -213,27 +220,29 @@ finalize_it: } -/* Helper to doSQLEscape. This is called if doSQLEscape +/* Helper to doEscape. This is called if doEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can * NOT simply return the unmodified string because this * may cause SQL injection. But we also can not simply * abort the run, this would be a DoS. I think an appropriate - * measure is to remove the dangerous \' characters. We + * measure is to remove the dangerous \' characters (SQL). We * replace them by \", which will break the message and * signatures eventually present - but this is the * best thing we can do now (or does anybody * have a better idea?). rgerhards 2004-11-23 - * added support for "escapeMode" (so doSQLEscape for details). - * if mode = 1, then backslashes are changed to slashes. + * added support for escape mode (see doEscape for details). + * if mode = SQL_ESCAPE, then backslashes are changed to slashes. * rgerhards 2005-09-22 */ -static void doSQLEmergencyEscape(register uchar *p, int escapeMode) +static void doEmergencyEscape(register uchar *p, int mode) { while(*p) { - if(*p == '\'') + if((mode == SQL_ESCAPE||mode == STDSQL_ESCAPE) && *p == '\'') *p = '"'; - else if((escapeMode == 1) && (*p == '\\')) + else if((mode == JSON_ESCAPE) && *p == '"') + *p = '\''; + else if((mode == SQL_ESCAPE) && *p == '\\') *p = '/'; ++p; } @@ -258,14 +267,16 @@ static void doSQLEmergencyEscape(register uchar *p, int escapeMode) * smartness depends on config settings. So we add a new option to this * function that allows the caller to select if they want to standard or * "smart" encoding ;) - * new parameter escapeMode is 0 - standard sql, 1 - "smart" engines + * -- + * Parameter "mode" is STDSQL_ESCAPE, SQL_ESCAPE "smart" SQL engines, or + * JSON_ESCAPE for everyone requiring escaped JSON (e.g. ElasticSearch). * 2005-09-22 rgerhards */ rsRetVal -doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode) +doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int mode) { DEFiRet; - uchar *p; + uchar *p = NULL; int iLen; cstr_t *pStrB = NULL; uchar *pszGenerated; @@ -276,26 +287,32 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM assert(pbMustBeFreed != NULL); /* first check if we need to do anything at all... */ - if(escapeMode == 0) + if(mode == STDSQL_ESCAPE) for(p = *pp ; *p && *p != '\'' ; ++p) ; - else + else if(mode == SQL_ESCAPE) for(p = *pp ; *p && *p != '\'' && *p != '\\' ; ++p) ; + else if(mode == JSON_ESCAPE) + for(p = *pp ; *p && *p != '"' ; ++p) + ; /* when we get out of the loop, we are either at the - * string terminator or the first \'. */ - if(*p == '\0') + * string terminator or the first character to escape */ + if(p && *p == '\0') FINALIZE; /* nothing to do in this case! */ p = *pp; iLen = *pLen; CHKiRet(cstrConstruct(&pStrB)); - + while(*p) { - if(*p == '\'') { - CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + if((mode == SQL_ESCAPE || mode == STDSQL_ESCAPE) && *p == '\'') { + CHKiRet(cstrAppendChar(pStrB, (mode == STDSQL_ESCAPE) ? '\'' : '\\')); + iLen++; /* reflect the extra character */ + } else if((mode == SQL_ESCAPE) && *p == '\\') { + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ - } else if((escapeMode == 1) && (*p == '\\')) { + } else if((mode == JSON_ESCAPE) && *p == '"') { CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } @@ -314,7 +331,7 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM finalize_it: if(iRet != RS_RET_OK) { - doSQLEmergencyEscape(*pp, escapeMode); + doEmergencyEscape(*pp, mode); if(pStrB != NULL) cstrDestruct(&pStrB); } @@ -883,11 +900,14 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) * acknowledged implementing the option. -- rgerhards, 2011-03-21 */ if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) { - pTpl->optFormatForSQL = 2; - DBGPRINTF("strgen suports the stdsql option\n"); + pTpl->optFormatEscape = STDSQL_ESCAPE; + DBGPRINTF("strgen supports the stdsql option\n"); } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) { - pTpl->optFormatForSQL = 1; - DBGPRINTF("strgen suports the sql option\n"); + pTpl->optFormatEscape = SQL_ESCAPE; + DBGPRINTF("strgen supports the sql option\n"); + } else if(lenMod > 4 && !strcasecmp((char*) szMod+ lenMod - 4, ",json")) { + pTpl->optFormatEscape = JSON_ESCAPE; + DBGPRINTF("strgen supports the json option\n"); } finalize_it: @@ -1015,11 +1035,13 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) * it anyhow... ;) rgerhards 2004-11-22 */ if(!strcmp(optBuf, "stdsql")) { - pTpl->optFormatForSQL = 2; + pTpl->optFormatEscape = STDSQL_ESCAPE; + } else if(!strcmp(optBuf, "json")) { + pTpl->optFormatEscape = JSON_ESCAPE; } else if(!strcmp(optBuf, "sql")) { - pTpl->optFormatForSQL = 1; + pTpl->optFormatEscape = SQL_ESCAPE; } else if(!strcmp(optBuf, "nosql")) { - pTpl->optFormatForSQL = 0; + pTpl->optFormatEscape = NO_ESCAPE; } else { dbgprintf("Invalid option '%s' ignored.\n", optBuf); } @@ -1180,9 +1202,11 @@ void tplPrintList(void) pTpl = tplRoot; while(pTpl != NULL) { dbgprintf("Template: Name='%s' ", pTpl->pszName == NULL? "NULL" : pTpl->pszName); - if(pTpl->optFormatForSQL == 1) + if(pTpl->optFormatEscape == SQL_ESCAPE) dbgprintf("[SQL-Format (MySQL)] "); - else if(pTpl->optFormatForSQL == 2) + else if(pTpl->optFormatEscape == JSON_ESCAPE) + dbgprintf("[JSON-Escaped Format] "); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) dbgprintf("[SQL-Format (standard SQL)] "); dbgprintf("\n"); pTpe = pTpl->pEntryRoot; diff --git a/template.h b/template.h index f7ac2e08..b5598b6d 100644 --- a/template.h +++ b/template.h @@ -36,9 +36,10 @@ struct template { int tpenElements; /* number of elements in templateEntry list */ struct templateEntry *pEntryRoot; struct templateEntry *pEntryLast; - char optFormatForSQL; /* in text fields, 0 - do not escape, - * 1 - escape quotes by double quotes, - * 2 - escape "the MySQL way" + char optFormatEscape; /* in text fields, 0 - do not escape, + * 1 - escape "the MySQL way" + * 2 - escape quotes by double quotes, + * 3 - escape double quotes for JSON. */ /* following are options. All are 0/1 defined (either on or off). * we use chars because they are faster than bit fields and smaller @@ -130,7 +131,7 @@ rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); */ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); -rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); +rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); -- cgit From f8019d52f83884acb5e8f8755fe976d1592b4ccb Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Mon, 15 Aug 2011 22:22:13 +1000 Subject: add elasticsearch output module Add support for sending events to elasticsearch - a distributed, RESTful, search engine built on Lucene (www.elasticsearch.org). The output module is enabled via a configure option, and uses libcurl to send the messages from rsyslog to elasticsearch. This patch makes use of the earlier JSON quoting patch to ensure valid JSON strings are sent to the server. Signed-off-by: Nathan Scott --- Makefile.am | 5 + configure.ac | 36 ++++ plugins/omelasticsearch/Makefile.am | 8 + plugins/omelasticsearch/omelasticsearch.c | 271 ++++++++++++++++++++++++++++++ tools/syslogd.c | 3 + 5 files changed, 323 insertions(+) create mode 100644 plugins/omelasticsearch/Makefile.am create mode 100644 plugins/omelasticsearch/omelasticsearch.c diff --git a/Makefile.am b/Makefile.am index de4777b2..fd348a67 100644 --- a/Makefile.am +++ b/Makefile.am @@ -168,6 +168,10 @@ if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif +if ENABLE_ELASTICSEARCH +SUBDIRS += plugins/omelasticsearch +endif + if ENABLE_MMSNMPTRAPD SUBDIRS += plugins/mmsnmptrapd endif @@ -244,5 +248,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-imtemplate \ --enable-omtemplate \ --enable-mmsnmptrapd \ + --enable-elasticsearch \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 6fbc84a6..099b78b9 100644 --- a/configure.ac +++ b/configure.ac @@ -669,6 +669,40 @@ AC_SUBST(SNMP_CFLAGS) AC_SUBST(SNMP_LIBS) +# elasticsearch support +AC_ARG_ENABLE(elasticsearch, + [AS_HELP_STRING([--enable-elasticsearch],[Enable elasticsearch output module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_elasticsearch="yes" ;; + no) enable_elasticsearch="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-elasticsearch) ;; + esac], + [enable_elasticsearch=no] +) +if test "x$enable_elasticsearch" = "xyes"; then + AC_CHECK_PROG( + [HAVE_CURL_CONFIG], + [curl-config], + [yes],,, + ) + if test "x${HAVE_CURL_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([curl-config not found in PATH]) + fi + AC_CHECK_LIB( + [curl], + [curl_global_init], + [CURL_CFLAGS="`curl-config --cflags`" + CURL_LIBS="`curl-config --libs`" + ], + [AC_MSG_FAILURE([curl library is missing])], + [`curl-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ELASTICSEARCH, test x$enable_elasticsearch = xyes) +AC_SUBST(CURL_CFLAGS) +AC_SUBST(CURL_LIBS) + + # GnuTLS support AC_ARG_ENABLE(gnutls, [AS_HELP_STRING([--enable-gnutls],[Enable GNU TLS support @<:@default=no@:>@])], @@ -1205,6 +1239,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ plugins/omudpspoof/Makefile \ + plugins/omelasticsearch/Makefile \ plugins/sm_cust_bindcdr/Makefile \ plugins/mmsnmptrapd/Makefile \ plugins/cust1/Makefile \ @@ -1239,6 +1274,7 @@ echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" echo " omhdfs module will be compiled: $enable_omhdfs" +echo " omelasticsearch module will be compiled: $enable_elasticsearch" echo " omruleset module will be compiled: $enable_omruleset" echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" diff --git a/plugins/omelasticsearch/Makefile.am b/plugins/omelasticsearch/Makefile.am new file mode 100644 index 00000000..a574c72f --- /dev/null +++ b/plugins/omelasticsearch/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omelasticsearch.la + +omelasticsearch_la_SOURCES = omelasticsearch.c +omelasticsearch_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omelasticsearch_la_LDFLAGS = -module -avoid-version +omelasticsearch_la_LIBADD = $(CURL_LIBS) + +EXTRA_DIST = diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c new file mode 100644 index 00000000..ce2e0c1f --- /dev/null +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -0,0 +1,271 @@ +/* omelasticsearch.c + * This is the http://www.elasticsearch.org/ output module. + * + * NOTE: read comments in module-template.h for more specifics! + * + * Copyright 2011 Nathan Scott. + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "statsobj.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP + +/* internal structures */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(statsobj) + +statsobj_t *indexStats; +STATSCOUNTER_DEF(indexConFail, mutIndexConFail) +STATSCOUNTER_DEF(indexSubmit, mutIndexSubmit) +STATSCOUNTER_DEF(indexFailed, mutIndexFailed) +STATSCOUNTER_DEF(indexSuccess, mutIndexSuccess) + +/* REST API for elasticsearch hits this URL: + * http://:// + */ +typedef struct curl_slist HEADER; +typedef struct _instanceData { + CURL *curlHandle; /* libcurl session handle */ + HEADER *postHeader; /* json POST request info */ +} instanceData; + +/* config variables */ +static int restPort = 9200; +static char *hostName = "localhost"; +static char *searchIndex = "system"; +static char *searchType = "events"; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +BEGINfreeInstance +CODESTARTfreeInstance + if (pData->postHeader) { + curl_slist_free_all(pData->postHeader); + pData->postHeader = NULL; + } + if (pData->curlHandle) { + curl_easy_cleanup(pData->curlHandle); + pData->curlHandle = NULL; + } +ENDfreeInstance + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +rsRetVal +curlPost(instanceData *instance, uchar *message) +{ + CURLcode code; + CURL *curl = instance->curlHandle; + int length = strlen((char *)message); + + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, length); + code = curl_easy_perform(curl); + switch (code) { + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_CONNECT: + case CURLE_WRITE_ERROR: + STATSCOUNTER_INC(indexConFail, mutIndexConFail); + return RS_RET_SUSPENDED; + default: + STATSCOUNTER_INC(indexSubmit, mutIndexSubmit); + return RS_RET_OK; + } +} + +BEGINdoAction +CODESTARTdoAction + CHKiRet(curlPost(pData, ppString[0])); +finalize_it: +ENDdoAction + +/* elasticsearch POST result string ... useful for debugging */ +size_t +curlResult(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + unsigned int i; + char *p = (char *)ptr; + char *jsonData = (char *)userdata; + static char ok[] = "{\"ok\":true,"; + + ASSERT(size == 1); + + if (size == 1 && + nmemb > sizeof(ok)-1 && + strncmp(p, ok, sizeof(ok)-1) == 0) { + STATSCOUNTER_INC(indexSuccess, mutIndexSuccess); + } else { + STATSCOUNTER_INC(indexFailed, mutIndexFailed); + if (Debug) { + DBGPRINTF("omelasticsearch request: %s\n", jsonData); + DBGPRINTF("omelasticsearch result: "); + for (i = 0; i < nmemb; i++) + DBGPRINTF("%c", p[i]); + DBGPRINTF("\n"); + } + } + return size * nmemb; +} + +static rsRetVal +curlSetup(instanceData *instance) +{ + char restURL[2048]; /* libcurl makes a copy, using the stack here is OK */ + HEADER *header; + CURL *handle; + + handle = curl_easy_init(); + if (handle == NULL) { + return RS_RET_OBJ_CREATION_FAILED; + } + + snprintf(restURL, sizeof(restURL)-1, "http://%s:%d/%s/%s", + hostName, restPort, searchIndex, searchType); + header = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8"); + + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult); + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header); + curl_easy_setopt(handle, CURLOPT_URL, restURL); + curl_easy_setopt(handle, CURLOPT_POST, 1); + + instance->curlHandle = handle; + instance->postHeader = header; + + DBGPRINTF("omelasticsearch setup, using REST URL: %s\n", restURL); + return RS_RET_OK; +} + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + if(strncmp((char*) p, ":omelasticsearch:", sizeof(":omelasticsearch:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + p += sizeof(":omelasticsearch:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (uchar*) " StdJSONFmt")); + + /* all good, we can now initialise our private data */ + CHKiRet(curlSetup(pData)); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINmodExit +CODESTARTmodExit + curl_global_cleanup(); + statsobj.Destruct(&indexStats); + objRelease(errmsg, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); +ENDmodExit + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + restPort = 9200; + hostName = "localhost"; + searchIndex = "system"; + searchType = "events"; + RETiRet; +} + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); + + /* register config file handlers */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchindex", 0, eCmdHdlrGetWord, NULL, &searchIndex, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchtype", 0, eCmdHdlrGetWord, NULL, &searchType, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchhost", 0, eCmdHdlrGetWord, NULL, &hostName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchport", 0, eCmdHdlrInt, NULL, &restPort, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + + if (curl_global_init(CURL_GLOBAL_ALL) != 0) { + errmsg.LogError(0, RS_RET_OBJ_CREATION_FAILED, "CURL fail. -elasticsearch indexing disabled"); + ABORT_FINALIZE(RS_RET_OBJ_CREATION_FAILED); + } + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&indexStats)); + CHKiRet(statsobj.SetName(indexStats, (uchar *)"elasticsearch")); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"connfail", + ctrType_IntCtr, &indexConFail)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"submits", + ctrType_IntCtr, &indexSubmit)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed", + ctrType_IntCtr, &indexFailed)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"success", + ctrType_IntCtr, &indexSuccess)); + CHKiRet(statsobj.ConstructFinalize(indexStats)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 0b7bbc96..bec2f9a7 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -334,6 +334,7 @@ static uchar template_WallFmt[] = "\"\r\n\7Message from syslogd@%HOSTNAME% at %t static uchar template_StdUsrMsgFmt[] = "\" %syslogtag%%msg%\n\r\""; static uchar template_StdDBFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-mysql%', '%timegenerated:::date-mysql%', %iut%, '%syslogtag%')\",SQL"; static uchar template_StdPgSQLFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%')\",STDSQL"; +static uchar template_StdJSONFmt[] = "\"{\\\"message\\\":\\\"%msg%\\\",\\\"fromhost\\\":\\\"%HOSTNAME%\\\",\\\"facility\\\":\\\"%syslogfacility-text%\\\",\\\"priority\\\":\\\"%syslogpriority-text%\\\",\\\"timereported\\\":\\\"%timereported:::date-rfc3339%\\\",\\\"timegenerated\\\":\\\"%timegenerated:::date-rfc3339%\\\"}\",JSON"; static uchar template_spoofadr[] = "\"%fromhost-ip%\""; /* end templates */ @@ -2173,6 +2174,8 @@ static rsRetVal mainThread() tplAddLine(" StdDBFmt", &pTmp); pTmp = template_StdPgSQLFmt; tplAddLine(" StdPgSQLFmt", &pTmp); + pTmp = template_StdJSONFmt; + tplAddLine(" StdJSONFmt", &pTmp); pTmp = template_spoofadr; tplLastStaticInit(tplAddLine("RSYSLOG_omudpspoofDfltSourceTpl", &pTmp)); -- cgit From 9911dacf59e898bae2c8c6619b85348bf54ddc23 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 Jan 2012 10:42:04 +0100 Subject: elasticsearch: move to asl 2.0 email conversation Nathan/Rainer 2012-01-16&17 --- plugins/omelasticsearch/omelasticsearch.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index ce2e0c1f..3bec1838 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -8,20 +8,19 @@ * * This file is part of rsyslog. * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "config.h" #include "rsyslog.h" -- cgit From 19a09944b477819c74b16c9861f1e7db085b4170 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 18 Jan 2012 10:57:55 +0100 Subject: imptcp: added basic stats counters --- plugins/imptcp/imptcp.c | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 65fe703c..951b24e4 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -65,6 +65,7 @@ #include "datetime.h" #include "ruleset.h" #include "msg.h" +#include "statsobj.h" #include "net.h" /* for permittedPeers, may be removed when this is removed */ /* the define is from tcpsrv.h, we need to find a new (but easier!!!) abstraction layer some time ... */ @@ -82,7 +83,7 @@ DEFobjCurrIf(prop) DEFobjCurrIf(datetime) DEFobjCurrIf(errmsg) DEFobjCurrIf(ruleset) - +DEFobjCurrIf(statsobj) /* config settings */ @@ -123,7 +124,8 @@ struct ptcpsrv_s { * includes support for doubly-linked list. */ struct ptcpsess_s { - ptcpsrv_t *pSrv; /* our server */ +// ptcpsrv_t *pSrv; /* our server TODO: check remove! */ + ptcplstn_t *pLstn; /* our listener */ ptcpsess_t *prev, *next; int sock; epolld_t *epd; @@ -151,6 +153,8 @@ struct ptcplstn_s { ptcplstn_t *prev, *next; int sock; epolld_t *epd; + statsobj_t *stats; /* listener stats */ + STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) }; @@ -492,22 +496,25 @@ static rsRetVal doSubmitMsg(ptcpsess_t *pThis, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) { msg_t *pMsg; + ptcpsrv_t *pSrv; DEFiRet; if(pThis->iMsg == 0) { DBGPRINTF("discarding zero-sized message\n"); FINALIZE; } + pSrv = pThis->pLstn->pSrv; /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); - MsgSetInputName(pMsg, pThis->pSrv->pInputName); + MsgSetInputName(pMsg, pSrv->pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; MsgSetRcvFrom(pMsg, pThis->peerName); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->peerIP)); - MsgSetRuleset(pMsg, pThis->pSrv->pRuleset); + MsgSetRuleset(pMsg, pSrv->pRuleset); + STATSCOUNTER_INC(pThis->pLstn->ctrSubmit, pThis->pLstn->mutCtrSubmit); if(pMultiSub == NULL) { CHKiRet(submitMsg(pMsg)); @@ -589,7 +596,8 @@ processDataRcvd(ptcpsess_t *pThis, char c, struct syslogTime *stTime, time_t ttG } if(( (c == '\n') - || ((pThis->pSrv->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->iAddtlFrameDelim)) + || ((pThis->pLstn->pSrv->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) + && (c == pThis->pLstn->pSrv->iAddtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); pThis->inputState = eAtStrtFram; @@ -751,10 +759,19 @@ addLstn(ptcpsrv_t *pSrv, int sock) { DEFiRet; ptcplstn_t *pLstn; + uchar statname[64]; CHKmalloc(pLstn = malloc(sizeof(ptcplstn_t))); pLstn->pSrv = pSrv; pLstn->sock = sock; + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&(pLstn->stats))); + snprintf((char*)statname, sizeof(statname), "imptcp(%s)", pSrv->port); + statname[sizeof(statname)-1] = '\0'; /* just to be on the save side... */ + CHKiRet(statsobj.SetName(pLstn->stats, statname)); + CHKiRet(statsobj.AddCounter(pLstn->stats, UCHAR_CONSTANT("submitted"), + ctrType_IntCtr, &(pLstn->ctrSubmit))); + CHKiRet(statsobj.ConstructFinalize(pLstn->stats)); /* add to start of server's listener list */ pLstn->prev = NULL; @@ -773,14 +790,15 @@ finalize_it: /* add a session to the server */ static rsRetVal -addSess(ptcpsrv_t *pSrv, int sock, prop_t *peerName, prop_t *peerIP) +addSess(ptcplstn_t *pLstn, int sock, prop_t *peerName, prop_t *peerIP) { DEFiRet; ptcpsess_t *pSess = NULL; + ptcpsrv_t *pSrv = pLstn->pSrv; CHKmalloc(pSess = malloc(sizeof(ptcpsess_t))); CHKmalloc(pSess->pMsg = malloc(iMaxLine * sizeof(uchar))); - pSess->pSrv = pSrv; + pSess->pLstn = pLstn; pSess->sock = sock; pSess->inputState = eAtStrtFram; pSess->iMsg = 0; @@ -824,7 +842,7 @@ closeSess(ptcpsess_t *pSess) pSess->next->prev = pSess->prev; if(pSess->prev == NULL) { /* need to update root! */ - pSess->pSrv->pSess = pSess->next; + pSess->pLstn->pSrv->pSess = pSess->next; } else { pSess->prev->next = pSess->next; } @@ -949,7 +967,7 @@ lstnActivity(ptcplstn_t *pLstn) if(localRet == RS_RET_NO_MORE_DATA) break; CHKiRet(localRet); - CHKiRet(addSess(pLstn->pSrv, newSock, peerName, peerIP)); + CHKiRet(addSess(pLstn, newSock, peerName, peerIP)); } finalize_it: @@ -979,7 +997,7 @@ sessActivity(ptcpsess_t *pSess) CHKiRet(DataRcvd(pSess, rcvBuf, lenRcv)); } else if (lenRcv == 0) { /* session was closed, do clean-up */ - if(pSess->pSrv->bEmitMsgOnClose) { + if(pSess->pLstn->pSrv->bEmitMsgOnClose) { uchar *peerName; int lenPeer; prop.GetString(pSess->peerName, &peerName, &lenPeer); @@ -1085,6 +1103,8 @@ shutdownSrv(ptcpsrv_t *pSrv) pLstn = pSrv->pLstn; while(pLstn != NULL) { close(pLstn->sock); + statsobj.Destruct(&(pLstn->stats)); + /* now unlink listner */ lstnDel = pLstn; pLstn = pLstn->next; DBGPRINTF("imptcp shutdown listen socket %d\n", lstnDel->sock); @@ -1132,6 +1152,7 @@ CODESTARTmodExit /* release objects we used */ objRelease(glbl, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); objRelease(datetime, CORE_COMPONENT); @@ -1167,6 +1188,7 @@ CODEmodInit_QueryRegCFSLineHdlr initConfigSettings(); /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); -- cgit From b41fc354ede3baec9740cb7e6a2f609824269836 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 18 Jan 2012 12:31:01 +0100 Subject: imptcp: added additional information to stats record --- plugins/imptcp/imptcp.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 951b24e4..103da210 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -192,7 +192,7 @@ static char rcvBuf[128*1024]; /* forward definitions */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal); -static rsRetVal addLstn(ptcpsrv_t *pSrv, int sock); +static rsRetVal addLstn(ptcpsrv_t *pSrv, int sock, int isIPv6); /* some simple constructors/destructors */ @@ -237,6 +237,7 @@ startupSrv(ptcpsrv_t *pSrv) int sockflags; struct addrinfo hints, *res = NULL, *r; uchar *lstnIP; + int isIPv6 = 0; lstnIP = pSrv->lstnIP == NULL ? UCHAR_CONSTANT("") : pSrv->lstnIP; @@ -269,8 +270,9 @@ startupSrv(ptcpsrv_t *pSrv) continue; } -#ifdef IPV6_V6ONLY if(r->ai_family == AF_INET6) { + isIPv6 = 1; +#ifdef IPV6_V6ONLY int iOn = 1; if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&iOn, sizeof (iOn)) < 0) { @@ -278,8 +280,8 @@ startupSrv(ptcpsrv_t *pSrv) sock = -1; continue; } - } #endif + } if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on)) < 0 ) { DBGPRINTF("error %d setting tcp socket option\n", errno); close(sock); @@ -341,7 +343,7 @@ startupSrv(ptcpsrv_t *pSrv) /* if we reach this point, we were able to obtain a valid socket, so we can * create our listener object. -- rgerhards, 2010-08-10 */ - CHKiRet(addLstn(pSrv, sock)); + CHKiRet(addLstn(pSrv, sock, isIPv6)); ++numSocks; } @@ -755,7 +757,7 @@ finalize_it: /* add a listener to the server */ static rsRetVal -addLstn(ptcpsrv_t *pSrv, int sock) +addLstn(ptcpsrv_t *pSrv, int sock, int isIPv6) { DEFiRet; ptcplstn_t *pLstn; @@ -766,7 +768,9 @@ addLstn(ptcpsrv_t *pSrv, int sock) pLstn->sock = sock; /* support statistics gathering */ CHKiRet(statsobj.Construct(&(pLstn->stats))); - snprintf((char*)statname, sizeof(statname), "imptcp(%s)", pSrv->port); + snprintf((char*)statname, sizeof(statname), "imptcp(%s/%s/%s)", + (pSrv->lstnIP == NULL) ? "*" : (char*)pSrv->lstnIP, pSrv->port, + isIPv6 ? "IPv6" : "IPv4"); statname[sizeof(statname)-1] = '\0'; /* just to be on the save side... */ CHKiRet(statsobj.SetName(pLstn->stats, statname)); CHKiRet(statsobj.AddCounter(pLstn->stats, UCHAR_CONSTANT("submitted"), -- cgit