From aef0aba9cd00fb225d2803210586b86a13547ce0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Aug 2010 12:49:59 +0200 Subject: fix required for new engine --- plugins/imptcp/imptcp.c | 1 - 1 file changed, 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 732590a9..905067ea 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -494,7 +494,6 @@ doSubmitMsg(ptcpsess_t *pThis, struct syslogTime *stTime, time_t ttGenTime, mult MsgSetInputName(pMsg, pThis->pSrv->pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pThis->peerName); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->peerIP)); MsgSetRuleset(pMsg, pThis->pSrv->pRuleset); -- cgit From 41327992680cfb8dd602ea92d6902448dd66413d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 7 Sep 2010 13:06:04 +0200 Subject: acquire /dev/log socket optionally from systemd --- plugins/imuxsock/Makefile.am | 4 ++-- plugins/imuxsock/imuxsock.c | 50 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 7 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/Makefile.am b/plugins/imuxsock/Makefile.am index a2fe0baa..8803937f 100644 --- a/plugins/imuxsock/Makefile.am +++ b/plugins/imuxsock/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imuxsock.la -imuxsock_la_SOURCES = imuxsock.c +imuxsock_la_SOURCES = imuxsock.c ../../runtime/sd-daemon.c ../../runtime/sd-daemon.h imuxsock_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imuxsock_la_LDFLAGS = -module -avoid-version -imuxsock_la_LIBADD = +imuxsock_la_LIBADD = diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 046f12f0..db53fcb6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -47,6 +47,7 @@ #include "prop.h" #include "debug.h" #include "unlimited_select.h" +#include "sd-daemon.h" MODULE_TYPE_INPUT @@ -74,7 +75,7 @@ DEFobjCurrIf(prop) static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -static int startIndexUxLocalSockets; /* process funix from that index on (used to +static int startIndexUxLocalSockets; /* process funix from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup */ @@ -177,7 +178,7 @@ static rsRetVal discardFunixn(void) prop.Destruct(&(funixHName[i])); } } - + return RS_RET_OK; } @@ -190,6 +191,40 @@ static int create_unix_socket(const char *path, int bCreatePath) if (path[0] == '\0') return -1; + if (strcmp(path, _PATH_LOG) == 0) { + int r; + + /* Check whether an FD was passed in from systemd. If + * so, it's the /dev/log socket, so use it. */ + + r = sd_listen_fds(0); + if (r < 0) { + errmsg.LogError(-r, NO_ERRCODE, "Failed to acquire systemd socket"); + return -1; + } + + if (r > 1) { + errmsg.LogError(EINVAL, NO_ERRCODE, "Wrong number of systemd sockets passed"); + return -1; + } + + if (r == 1) { + fd = SD_LISTEN_FDS_START; + r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); + if (r < 0) { + errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); + return -1; + } + + if (!r) { + errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); + return -1; + } + + return fd; + } + } + unlink(path); memset(&sunx, 0, sizeof(sunx)); @@ -391,12 +426,17 @@ CODESTARTafterRun int i; /* do cleanup here */ /* Close the UNIX sockets. */ - for (i = 0; i < nfunix; i++) + for (i = 0; i < nfunix; i++) if (funix[i] != -1) close(funix[i]); - /* Clean-up files. */ - for(i = startIndexUxLocalSockets; i < nfunix; i++) + /* Clean-up files. If systemd passed us a socket it is + * systemd's job to clean it up.*/ + if (sd_listen_fds(0) > 0) + i = 1; + else + i = startIndexUxLocalSockets; + for(; i < nfunix; i++) if (funixn[i] && funix[i] != -1) unlink((char*) funixn[i]); /* free no longer needed string */ -- cgit From 21a2672222a1b403ff0811154dec156676bed732 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Sep 2010 12:46:03 +0200 Subject: moved systemd interface to rsyslog convenience lib Mostly a refresh of sd-daemon.[ch] from its source plus some make file changes. We now have systemd interfaces inside rsyslog, so that all plugins interested can call the interfaces. Seems not to be totally necessary right now, but will help in the long term. --- plugins/imuxsock/Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/Makefile.am b/plugins/imuxsock/Makefile.am index 8803937f..28f9f9e3 100644 --- a/plugins/imuxsock/Makefile.am +++ b/plugins/imuxsock/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imuxsock.la -imuxsock_la_SOURCES = imuxsock.c ../../runtime/sd-daemon.c ../../runtime/sd-daemon.h -imuxsock_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +imuxsock_la_SOURCES = imuxsock.c +imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imuxsock_la_LDFLAGS = -module -avoid-version -imuxsock_la_LIBADD = +imuxsock_la_LIBADD = $(RSRT_LIBS) -- cgit From 3cbd7300c47991aadca00db1b028fd3d5d551040 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Sep 2010 12:24:22 +0200 Subject: added module impstat to emit periodic statistics on rsyslog counters This is a *very first* and *very rough* and *very featureless* first shot at this functionality. It is assumed that we will enhance the stats system as a by-line while doing other development. --- plugins/impstats/Makefile.am | 6 ++ plugins/impstats/impstats.c | 211 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 plugins/impstats/Makefile.am create mode 100644 plugins/impstats/impstats.c (limited to 'plugins') diff --git a/plugins/impstats/Makefile.am b/plugins/impstats/Makefile.am new file mode 100644 index 00000000..187bbca4 --- /dev/null +++ b/plugins/impstats/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = impstats.la + +impstats_la_SOURCES = impstats.c +impstats_la_CPPFLAGS = $(RSRT_CFLAGS) -I$(top_srcdir) $(PTHREADS_CFLAGS) +impstats_la_LDFLAGS = -module -avoid-version +impstats_la_LIBADD = diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c new file mode 100644 index 00000000..5f0d19eb --- /dev/null +++ b/plugins/impstats/impstats.c @@ -0,0 +1,211 @@ +/* impstats.c + * A module to periodically output statistics gathered by rsyslog. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c) + * This file is under development and has not yet arrived at being fully + * self-contained and a real object. So far, it is mostly an excerpt + * of the "old" message code without any modifications. However, it + * helps to have things at the right place one we go to the meat of it. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "cfsysline.h" +#include "module-template.h" +#include "errmsg.h" +#include "msg.h" +#include "srUtils.h" +#include "unicode-helper.h" +#include "glbl.h" +#include "prop.h" + +MODULE_TYPE_INPUT + +/* defines */ +#define DEFAULT_STATS_PERIOD (5 * 60) +#define DEFAULT_FACILITY 5 /* syslog */ +#define DEFAULT_SEVERITY 6 /* info */ + +/* Module static data */ +DEF_IMOD_STATIC_DATA +DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) +typedef struct configSettings_s { + int iStatsInterval; + int iFacility; + int iSeverity; +} configSettings_t; + +static configSettings_t cs; + +static prop_t *pInputName = NULL; +static prop_t *pLocalHostIP = NULL; + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +static inline void +initConfigSettings(void) +{ + cs.iStatsInterval = DEFAULT_STATS_PERIOD; + cs.iFacility = DEFAULT_FACILITY; + cs.iSeverity = DEFAULT_SEVERITY; +} + + +/* actually submit a message to the rsyslog core + */ +static inline rsRetVal +doSubmitMsg(uchar *line) +{ + msg_t *pMsg; + DEFiRet; + + CHKiRet(msgConstruct(&pMsg)); + MsgSetInputName(pMsg, pInputName); + MsgSetRawMsgWOSize(pMsg, (char*)line); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); + MsgSetRcvFromIP(pMsg, pLocalHostIP); + MsgSetMSGoffs(pMsg, 0); + MsgSetTAG(pMsg, UCHAR_CONSTANT("rsyslogd-pstats:"), sizeof("rsyslogd-pstats:") - 1); + pMsg->iFacility = cs.iFacility; + pMsg->iSeverity = cs.iSeverity; + pMsg->msgFlags = 0; + + submitMsg(pMsg); + +finalize_it: + RETiRet; + +} + + +/* the function to generate the actual statistics messages + * rgerhards, 2010-09-09 + */ +static inline void +generateStatsMsgs(void) +{ + int iMsgQueueSize; + uchar msg[1024]; + rsRetVal iRet; + + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + snprintf((char*)msg, sizeof(msg), "mainqueue size=%d", iMsgQueueSize); + doSubmitMsg(msg); + +finalize_it: + /*empty statement needed per C syntax*/; +} + + +BEGINrunInput +CODESTARTrunInput + /* this is an endless loop - it is terminated when the thread is + * signalled to do so. This, however, is handled by the framework, + * right into the sleep below. + */ + while(1) { +dbgprintf("impstats: stats interval %d seconds - going to sleep\n", cs.iStatsInterval); + srSleep(cs.iStatsInterval, 0); /* seconds, micro seconds */ +dbgprintf("impstats: woke up\n"); + + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + + generateStatsMsgs(); + } +ENDrunInput + + +BEGINwillRun +CODESTARTwillRun + DBGPRINTF("impstats: stats interval %d seconds\n", cs.iStatsInterval); + if(cs.iStatsInterval == 0) + ABORT_FINALIZE(RS_RET_NO_RUN); +finalize_it: +ENDwillRun + + +BEGINafterRun +CODESTARTafterRun +ENDafterRun + + +BEGINmodExit +CODESTARTmodExit + /* release objects we used */ + objRelease(glbl, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); +ENDmodExit + + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + initConfigSettings(); + return RS_RET_OK; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + DBGPRINTF("impstats version %s loading\n", VERSION); + initConfigSettings(); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatsinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatfacility", 0, eCmdHdlrInt, NULL, &cs.iFacility, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatseverity", 0, eCmdHdlrInt, NULL, &cs.iSeverity, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("rsyslogd-pstats"), sizeof("rsyslogd-pstats") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + + CHKiRet(prop.Construct(&pLocalHostIP)); + CHKiRet(prop.SetString(pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pLocalHostIP)); +ENDmodInit +/* vi:set ai: + */ -- cgit From 3d50fd6a67b7d0bf31628aea14128a47dd5d326f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Sep 2010 12:37:11 +0200 Subject: fixed small memory leak, corrected input name --- plugins/impstats/impstats.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index 5f0d19eb..a818d20c 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -166,6 +166,8 @@ ENDafterRun BEGINmodExit CODESTARTmodExit + prop.Destruct(&pInputName); + prop.Destruct(&pLocalHostIP); /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); @@ -200,7 +202,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(prop.Construct(&pInputName)); - CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("rsyslogd-pstats"), sizeof("rsyslogd-pstats") - 1)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("impstats"), sizeof("impstats") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); CHKiRet(prop.Construct(&pLocalHostIP)); -- cgit From e86cb62f1299ef18732f7b3b87d45a840ee38f1e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 13 Sep 2010 15:43:56 +0200 Subject: improved statistics-gathering subsystem ... well, actually this is a first real implementation of this subsystem. I have added a counter registry, a way to access the countres (as readable string) and a way to define and maintem them. Also, module impstats has been updated to utilize the new system. Finally, I added some counters. I hope that this sets the baseline for useful future enhancements. --- plugins/impstats/impstats.c | 39 +++++++++++++++++++++++++++------------ plugins/imuxsock/imuxsock.c | 18 ++++++++++++++++++ 2 files changed, 45 insertions(+), 12 deletions(-) (limited to 'plugins') diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index a818d20c..1312a4e8 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -45,6 +45,7 @@ #include "srUtils.h" #include "unicode-helper.h" #include "glbl.h" +#include "statsobj.h" #include "prop.h" MODULE_TYPE_INPUT @@ -58,6 +59,9 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(statsobj) +DEFobjCurrIf(errmsg) + typedef struct configSettings_s { int iStatsInterval; int iFacility; @@ -112,22 +116,25 @@ finalize_it: } +/* callback for statsobj + * Note: usrptr exists only to satisfy requirements of statsobj callback interface! + */ +static rsRetVal +doStatsLine(void __attribute__((unused)) *usrptr, cstr_t *cstr) +{ + DEFiRet; + doSubmitMsg(rsCStrGetSzStrNoNULL(cstr)); + RETiRet; +} + + /* the function to generate the actual statistics messages * rgerhards, 2010-09-09 */ static inline void generateStatsMsgs(void) { - int iMsgQueueSize; - uchar msg[1024]; - rsRetVal iRet; - - CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); - snprintf((char*)msg, sizeof(msg), "mainqueue size=%d", iMsgQueueSize); - doSubmitMsg(msg); - -finalize_it: - /*empty statement needed per C syntax*/; + statsobj.GetAllStatsLines(doStatsLine, NULL); } @@ -138,9 +145,7 @@ CODESTARTrunInput * right into the sleep below. */ while(1) { -dbgprintf("impstats: stats interval %d seconds - going to sleep\n", cs.iStatsInterval); srSleep(cs.iStatsInterval, 0); /* seconds, micro seconds */ -dbgprintf("impstats: woke up\n"); if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ @@ -151,10 +156,16 @@ ENDrunInput BEGINwillRun + rsRetVal localRet; CODESTARTwillRun DBGPRINTF("impstats: stats interval %d seconds\n", cs.iStatsInterval); if(cs.iStatsInterval == 0) ABORT_FINALIZE(RS_RET_NO_RUN); + localRet = statsobj.EnableStats(); + if(localRet != RS_RET_OK) { + errmsg.LogError(0, localRet, "impstat: error enabling statistics gathering"); + ABORT_FINALIZE(RS_RET_NO_RUN); + } finalize_it: ENDwillRun @@ -171,6 +182,8 @@ CODESTARTmodExit /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); ENDmodExit @@ -196,6 +209,8 @@ CODEmodInit_QueryRegCFSLineHdlr initConfigSettings(); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatsinterval", 0, eCmdHdlrInt, NULL, &cs.iStatsInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatfacility", 0, eCmdHdlrInt, NULL, &cs.iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"pstatseverity", 0, eCmdHdlrInt, NULL, &cs.iSeverity, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index db53fcb6..d976c676 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -48,6 +48,7 @@ #include "debug.h" #include "unlimited_select.h" #include "sd-daemon.h" +#include "statsobj.h" MODULE_TYPE_INPUT @@ -72,6 +73,10 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(statsobj) + +statsobj_t *modStats; +STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ @@ -269,6 +274,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); + STATSCOUNTER_INC(ctrSubmit, mutCtrSubmit); finalize_it: RETiRet; } @@ -453,9 +459,12 @@ ENDafterRun BEGINmodExit CODESTARTmodExit + statsobj.Destruct(&modStats); + objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); ENDmodExit @@ -502,6 +511,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); @@ -547,6 +557,14 @@ CODEmodInit_QueryRegCFSLineHdlr setSystemLogTimestampIgnore, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&modStats)); + CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), + ctrType_IntCtr, &ctrSubmit)); + CHKiRet(statsobj.ConstructFinalize(modStats)); + ENDmodInit /* vim:set ai: */ -- cgit From aa1b8f38c3611fd01710e2e3d0189c90cf9a7e9d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 18:05:12 +0200 Subject: first shot at SCM_CREDENTIALS --- plugins/imuxsock/imuxsock.c | 67 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index d976c676..9a245f88 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "dirty.h" #include "cfsysline.h" #include "unicode-helper.h" @@ -192,6 +193,7 @@ static int create_unix_socket(const char *path, int bCreatePath) { struct sockaddr_un sunx; int fd; + int one; if (path[0] == '\0') return -1; @@ -226,6 +228,7 @@ static int create_unix_socket(const char *path, int bCreatePath) return -1; } + //TOD: add logic to prepare socket (or at better place!) return fd; } } @@ -246,6 +249,24 @@ static int create_unix_socket(const char *path, int bCreatePath) close(fd); return -1; } + one = 1; +dbgprintf("imuxsock: setting socket options!\n"); + if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + close(fd); + return -1; + } + if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + close(fd); + return -1; + } + one = 1; + if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); + close(fd); + return -1; + } return fd; } @@ -264,6 +285,12 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + // TODO: here we need to mangle the raw message if we need to + // "fix up" the user pid. In the long term, it may make more sense + // to add this (somehow) to the message object (problem: at this stage, + // it is not fully available, eg no structured data). Alternatively, we + // may use a custom parser for our messages, but that doesn't play too well + // with the rest of the system. if(funixParseHost[iSock]) { pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; } else { @@ -292,7 +319,13 @@ static rsRetVal readSocket(int fd, int iSock) DEFiRet; int iRcvd; int iMaxLine; + struct msghdr msgh; + struct iovec msgiov; + static int ratelimitErrmsg = 0; // TODO: atomic OPS + struct cmsghdr *cm; + struct ucred *cred; uchar bufRcv[4096+1]; + char aux[1024]; uchar *pRcv = NULL; /* receive buffer */ assert(iSock >= 0); @@ -311,15 +344,43 @@ static rsRetVal readSocket(int fd, int iSock) CHKmalloc(pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))); } - iRcvd = recv(fd, pRcv, iMaxLine, 0); + memset(&msgh, 0, sizeof(msgh)); + memset(&msgiov, 0, sizeof(msgiov)); + memset(&aux, 0, sizeof(aux)); + msgiov.iov_base = pRcv; + msgiov.iov_len = iMaxLine; + msgh.msg_iov = &msgiov; + msgh.msg_iovlen = 1; + msgh.msg_control = aux; + msgh.msg_controllen = sizeof(aux); + iRcvd = recvmsg(fd, &msgh, MSG_DONTWAIT); +/* iRcvd = recv(fd, pRcv, iMaxLine, 0); + */ + dbgprintf("Message from UNIX socket: #%d\n", fd); if (iRcvd > 0) { + ratelimitErrmsg = 0; + + +dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); + for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { +dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { + cred = (struct ucred*) CMSG_DATA(cm); + dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); + //break; + } + } +dbgprintf("XXX: post CM loop\n"); + + + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); - } else if (iRcvd < 0 && errno != EINTR) { + } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("UNIX socket error: %d = %s.\n", errno, errStr); - errmsg.LogError(errno, NO_ERRCODE, "recvfrom UNIX"); + errmsg.LogError(errno, NO_ERRCODE, "imuxsock: recvfrom UNIX"); } finalize_it: -- cgit From c2d36c6894f7c36c2aed23c1e7ee9580c58659a7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 18:57:05 +0200 Subject: patching SCM_CREDENTIALS pid into TAG of imuxsock read messages --- plugins/imuxsock/imuxsock.c | 179 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 141 insertions(+), 38 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 9a245f88..fff6ac72 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -29,6 +29,7 @@ #include "rsyslog.h" #include #include +#include #include #include #include @@ -50,6 +51,7 @@ #include "unlimited_select.h" #include "sd-daemon.h" #include "statsobj.h" +#include "datetime.h" MODULE_TYPE_INPUT @@ -74,6 +76,7 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(datetime) DEFobjCurrIf(statsobj) statsobj_t *modStats; @@ -189,9 +192,38 @@ static rsRetVal discardFunixn(void) } -static int create_unix_socket(const char *path, int bCreatePath) +/* used to create a log socket if NOT passed in via systemd. + */ +static inline rsRetVal +createLogSocket(const char *path, int bCreatePath, int *fd) { struct sockaddr_un sunx; + DEFiRet; + + unlink(path); + memset(&sunx, 0, sizeof(sunx)); + sunx.sun_family = AF_UNIX; + if(bCreatePath) { + makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); + } + strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); + *fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if(*fd < 0 || bind(*fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || + chmod(path, 0666) < 0) { + errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); + dbgprintf("cannot create %s (%d).\n", path, errno); + close(*fd); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } +finalize_it: + RETiRet; +} + + +static inline rsRetVal +openLogSocket(const char *path, int bCreatePath, int *pfd) +{ + DEFiRet; int fd; int one; @@ -201,18 +233,18 @@ static int create_unix_socket(const char *path, int bCreatePath) if (strcmp(path, _PATH_LOG) == 0) { int r; - /* Check whether an FD was passed in from systemd. If + /* System log socket code. Check whether an FD was passed in from systemd. If * so, it's the /dev/log socket, so use it. */ r = sd_listen_fds(0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to acquire systemd socket"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (r > 1) { errmsg.LogError(EINVAL, NO_ERRCODE, "Wrong number of systemd sockets passed"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (r == 1) { @@ -220,46 +252,31 @@ static int create_unix_socket(const char *path, int bCreatePath) r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if (!r) { errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } - - //TOD: add logic to prepare socket (or at better place!) - return fd; - } - } - - unlink(path); - - memset(&sunx, 0, sizeof(sunx)); - sunx.sun_family = AF_UNIX; - if(bCreatePath) { - makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); - } - (void) strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); - fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if (fd < 0 || bind(fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || - chmod(path, 0666) < 0) { - errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); - dbgprintf("cannot create %s (%d).\n", path, errno); - close(fd); - return -1; + } else { + CHKiRet(createLogSocket(path, bCreatePath, &fd)); + } + } else { + CHKiRet(createLogSocket(path, bCreatePath, &fd)); } + one = 1; dbgprintf("imuxsock: setting socket options!\n"); - if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS create '%s'", path); + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } one = 1; if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { @@ -267,16 +284,53 @@ dbgprintf("imuxsock: setting socket options!\n"); close(fd); return -1; } - return fd; + *pfd = fd; +finalize_it: + RETiRet; +} + + +/* patch correct pid into tag. bufTAG MUST be CONF_TAG_MAXSIZE long! + */ +static inline void +fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) +{ + int i; + char bufPID[16]; + int lenPID; + + if(cred == NULL) + return; + + lenPID = snprintf(bufPID, sizeof(bufPID), "[%u]:", (unsigned) cred->pid); + + for(i = *lenTag ; i >= 0 && bufTAG[i] != '[' ; --i) + /*JUST SKIP*/; + + if(i < 0) + i = *lenTag - 1; /* go right at end of TAG, pid was not present (-1 for ':') */ + + if(i + lenPID > CONF_TAG_MAXSIZE) + return; /* do not touch, as things would break */ + + memcpy(bufTAG + i, bufPID, lenPID); + *lenTag = i + lenPID; } /* submit received message to the queue engine + * We now parse the message according to expected format so that we + * can also mangle it if necessary. */ static inline rsRetVal -SubmitMsg(uchar *pRcv, int lenRcv, int iSock) +SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) { msg_t *pMsg; + int lenMsg; + int i; + uchar *parse; + int pri; + uchar bufParseTAG[CONF_TAG_MAXSIZE]; DEFiRet; /* we now create our own message object and submit it to the queue */ @@ -285,6 +339,43 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); +// TODO: handle format errors + parse = pRcv; + lenMsg = lenRcv; + + parse++; lenMsg--; /* '<' */ + pri = 0; + while(lenMsg && isdigit(*parse)) { + pri = pri * 10 + *parse - '0'; + ++parse; + --lenMsg; + } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); +dbgprintf("imuxsock: submit: facil %d, sever %d\n", pMsg->iFacility, pMsg->iSeverity); + + parse++; lenMsg--; /* '>' */ + +dbgprintf("imuxsock: submit: stage 2\n"); + if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { + dbgprintf("we have a problem, invalid timestamp in msg!\n"); + } + + /* pull tag */ +dbgprintf("imuxsock: submit: stage 3\n"); + + i = 0; + while(lenMsg > 0 && *parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *parse++; + --lenMsg; + } + bufParseTAG[i] = '\0'; /* terminate string */ + fixPID(bufParseTAG, &i, cred); + MsgSetTAG(pMsg, bufParseTAG, i); + + MsgSetMSGoffs(pMsg, lenRcv - lenMsg); + // TODO: here we need to mangle the raw message if we need to // "fix up" the user pid. In the long term, it may make more sense // to add this (somehow) to the message object (problem: at this stage, @@ -292,9 +383,9 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock) // may use a custom parser for our messages, but that doesn't play too well // with the rest of the system. if(funixParseHost[iSock]) { - pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->msgFlags = funixFlags[iSock] | PARSE_HOSTNAME; } else { - pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING; + pMsg->msgFlags = funixFlags[iSock]; } MsgSetRcvFrom(pMsg, funixHName[iSock]); @@ -363,6 +454,7 @@ static rsRetVal readSocket(int fd, int iSock) dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); + cred = NULL; for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { @@ -375,7 +467,7 @@ dbgprintf("XXX: post CM loop\n"); - CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock, cred)); } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -458,6 +550,7 @@ ENDrunInput BEGINwillRun CODESTARTwillRun register int i; + int actSocks; /* first apply some config settings */ # ifdef OS_SOLARIS @@ -474,9 +567,17 @@ CODESTARTwillRun funixn[0] = pLogSockName; /* initialize and return if will run or not */ + actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfunix ; i++) { - if ((funix[i] = create_unix_socket((char*) funixn[i], funixCreateSockPath[i])) != -1) + if(openLogSocket((char*) funixn[i], funixCreateSockPath[i], &(funix[i])) == RS_RET_OK) { + ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); + } + } + + if(actSocks == 0) { + errmsg.LogError(0, NO_ERRCODE, "imuxsock does not run because we could not aquire any socket\n"); + ABORT_FINALIZE(RS_RET_ERR); } /* we need to create the inputName property (only once during our lifetime) */ @@ -526,6 +627,7 @@ CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); objRelease(statsobj, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); ENDmodExit @@ -573,6 +675,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(statsobj, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); -- cgit From 7d680cb7590d4eefa4631a249a02248b4168c7c6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 19:05:20 +0200 Subject: fixed nit --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index fff6ac72..496e679f 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -282,7 +282,7 @@ dbgprintf("imuxsock: setting socket options!\n"); if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); close(fd); - return -1; + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } *pfd = fd; finalize_it: -- cgit From 43933920353ed6911738255d509cd001960ff1b7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 10:28:47 +0200 Subject: changed imuxsock to have a somewhat better representation of listeners still not great, but far better readable (and extendable) than what we had before. --- plugins/imuxsock/imuxsock.c | 156 +++++++++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 61 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 496e679f..493f11c4 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -82,20 +82,26 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +/* structure to describe a specific listener */ +struct lstn_s { + uchar *sockName; /* read-only after startup */ + prop_t *hostName; /* host-name override - if set, use this instead of actual name */ + int fd; /* read-only after startup */ + int flags; /* should parser parse host name? read-only after startup */ + int flowCtl; /* flow control settings for this socket */ + sbool bParseHost; /* should parser parse host name? read-only after startup */ + sbool bUseCreds; + sbool bCreateSockPath; /* auto-creation of socket directory? */ +}; +static struct lstn_s listeners[MAXFUNIX]; + static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -static int startIndexUxLocalSockets; /* process funix from that index on (used to +static int startIndexUxLocalSockets; /* process fd from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup */ -static int funixParseHost[MAXFUNIX] = { 0, }; /* should parser parse host name? read-only after startup */ -static int funixFlags[MAXFUNIX] = { IGNDATE, }; /* should parser parse host name? read-only after startup */ -static int funixCreateSockPath[MAXFUNIX] = { 0, }; /* auto-creation of socket directory? */ -static uchar *funixn[MAXFUNIX] = { (uchar*) _PATH_LOG }; /* read-only after startup */ -static prop_t *funixHName[MAXFUNIX] = { NULL, }; /* host-name override - if set, use this instead of actual name */ -static int funixFlowCtl[MAXFUNIX] = { eFLOWCTL_NO_DELAY, }; /* flow control settings for this socket */ -static int funix[MAXFUNIX] = { -1, }; /* read-only after startup */ -static int nfunix = 1; /* number of Unix sockets open / read-only after startup */ +static int nfd = 1; /* number of Unix sockets open / read-only after startup */ /* config settings */ static int bOmitLocalLogging = 0; @@ -103,6 +109,7 @@ static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreateSockPath 0 static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? */ @@ -114,7 +121,7 @@ static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? * static rsRetVal setSystemLogTimestampIgnore(void __attribute__((unused)) *pVal, int iNewVal) { DEFiRet; - funixFlags[0] = iNewVal ? IGNDATE : NOFLAG; + listeners[0].flags = iNewVal ? IGNDATE : NOFLAG; RETiRet; } @@ -123,7 +130,7 @@ static rsRetVal setSystemLogTimestampIgnore(void __attribute__((unused)) *pVal, static rsRetVal setSystemLogFlowControl(void __attribute__((unused)) *pVal, int iNewVal) { DEFiRet; - funixFlowCtl[0] = iNewVal ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; + listeners[0].flowCtl = iNewVal ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; RETiRet; } @@ -140,26 +147,28 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; - if(nfunix < MAXFUNIX) { + if(nfd < MAXFUNIX) { if(*pNewVal == ':') { - funixParseHost[nfunix] = 1; + listeners[nfd].bParseHost = 1; } else { - funixParseHost[nfunix] = 0; + listeners[nfd].bParseHost = 0; } - CHKiRet(prop.Construct(&(funixHName[nfunix]))); + CHKiRet(prop.Construct(&(listeners[nfd].hostName))); if(pLogHostName == NULL) { - CHKiRet(prop.SetString(funixHName[nfunix], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.SetString(listeners[nfd].hostName, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); } else { - CHKiRet(prop.SetString(funixHName[nfunix], pLogHostName, ustrlen(pLogHostName))); + CHKiRet(prop.SetString(listeners[nfd].hostName, pLogHostName, ustrlen(pLogHostName))); /* reset hostname for next socket */ free(pLogHostName); pLogHostName = NULL; } - CHKiRet(prop.ConstructFinalize(funixHName[nfunix])); - funixFlowCtl[nfunix] = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; - funixFlags[nfunix] = bIgnoreTimestamp ? IGNDATE : NOFLAG; - funixCreateSockPath[nfunix] = bCreateSockPath; - funixn[nfunix++] = pNewVal; + CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); + listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; + listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; + listeners[nfd].bCreateSockPath = bCreateSockPath; + listeners[nfd].sockName = pNewVal; + listeners[nfd].bUseCreds = bUseCreds; + nfd++; } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", pNewVal); @@ -170,21 +179,21 @@ finalize_it: } -/* free the funixn[] socket names - needed as cleanup on several places - * note that nfunix is NOT reset! funixn[0] is never freed, as it comes from +/* free the sockName[] socket names - needed as cleanup on several places + * note that nfd is NOT reset! sockName[0] is never freed, as it comes from * the constant memory pool - and if not, it is freeed via some other pointer. */ static rsRetVal discardFunixn(void) { int i; - for (i = 1; i < nfunix; i++) { - if(funixn[i] != NULL) { - free(funixn[i]); - funixn[i] = NULL; + for (i = 1; i < nfd; i++) { + if(listeners[i].sockName != NULL) { + free(listeners[i].sockName); + listeners[i].sockName = NULL; } - if(funixHName[i] != NULL) { - prop.Destruct(&(funixHName[i])); + if(listeners[i].hostName != NULL) { + prop.Destruct(&(listeners[i].hostName)); } } @@ -267,7 +276,6 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) } one = 1; -dbgprintf("imuxsock: setting socket options!\n"); if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); close(fd); @@ -284,6 +292,17 @@ dbgprintf("imuxsock: setting socket options!\n"); close(fd); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } + + + +static socklen_t two =sizeof(one); + if(getsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one,&two) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); + close(fd); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } +dbgprintf("imuxsock: SO_TIMESTAMP is %d\n", one); + *pfd = fd; finalize_it: RETiRet; @@ -337,7 +356,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) CHKiRet(msgConstruct(&pMsg)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); - MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + MsgSetFlowControlType(pMsg, listeners[iSock].flowCtl); // TODO: handle format errors parse = pRcv; @@ -382,13 +401,13 @@ dbgprintf("imuxsock: submit: stage 3\n"); // it is not fully available, eg no structured data). Alternatively, we // may use a custom parser for our messages, but that doesn't play too well // with the rest of the system. - if(funixParseHost[iSock]) { - pMsg->msgFlags = funixFlags[iSock] | PARSE_HOSTNAME; + if(listeners[iSock].bParseHost) { + pMsg->msgFlags = listeners[iSock].flags | PARSE_HOSTNAME; } else { - pMsg->msgFlags = funixFlags[iSock]; + pMsg->msgFlags = listeners[iSock].flags; } - MsgSetRcvFrom(pMsg, funixHName[iSock]); + MsgSetRcvFrom(pMsg, listeners[iSock].hostName); CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); @@ -453,7 +472,7 @@ static rsRetVal readSocket(int fd, int iSock) ratelimitErrmsg = 0; -dbgprintf("XXX: pre CM loop, length of control message %d\n", msgh.msg_controllen); +dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); cred = NULL; for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); @@ -511,10 +530,11 @@ CODESTARTrunInput maxfds = 0; FD_ZERO (pReadfds); /* Copy master connections */ - for (i = startIndexUxLocalSockets; i < nfunix; i++) { - if (funix[i] != -1) { - FD_SET(funix[i], pReadfds); - if (funix[i]>maxfds) maxfds=funix[i]; + for (i = startIndexUxLocalSockets; i < nfd; i++) { + if (listeners[i].fd!= -1) { + FD_SET(listeners[i].fd, pReadfds); + if(listeners[i].fd > maxfds) + maxfds=listeners[i].fd; } } @@ -531,10 +551,10 @@ CODESTARTrunInput if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ - for (i = 0; i < nfunix && nfds > 0; i++) { + for (i = 0; i < nfd && nfds > 0; i++) { if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ - if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) { + if ((fd = listeners[i].fd) != -1 && FD_ISSET(fd, pReadfds)) { readSocket(fd, i); --nfds; /* indicate we have processed one */ } @@ -564,14 +584,15 @@ CODESTARTwillRun startIndexUxLocalSockets = bOmitLocalLogging ? 1 : 0; # endif if(pLogSockName != NULL) - funixn[0] = pLogSockName; + listeners[0].sockName = pLogSockName; /* initialize and return if will run or not */ actSocks = 0; - for (i = startIndexUxLocalSockets ; i < nfunix ; i++) { - if(openLogSocket((char*) funixn[i], funixCreateSockPath[i], &(funix[i])) == RS_RET_OK) { + for (i = startIndexUxLocalSockets ; i < nfd ; i++) { + if(openLogSocket((char*) listeners[i].sockName, listeners[i].bCreateSockPath, + &(listeners[i].fd)) == RS_RET_OK) { ++actSocks; - dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); + dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } } @@ -594,9 +615,9 @@ CODESTARTafterRun int i; /* do cleanup here */ /* Close the UNIX sockets. */ - for (i = 0; i < nfunix; i++) - if (funix[i] != -1) - close(funix[i]); + for (i = 0; i < nfd; i++) + if (listeners[i].fd != -1) + close(listeners[i].fd); /* Clean-up files. If systemd passed us a socket it is * systemd's job to clean it up.*/ @@ -604,15 +625,15 @@ CODESTARTafterRun i = 1; else i = startIndexUxLocalSockets; - for(; i < nfunix; i++) - if (funixn[i] && funix[i] != -1) - unlink((char*) funixn[i]); + for(; i < nfd; i++) + if (listeners[i].sockName && listeners[i].fd != -1) + unlink((char*) listeners[i].sockName); /* free no longer needed string */ free(pLogSockName); free(pLogHostName); discardFunixn(); - nfunix = 1; + nfd = 1; if(pInputName != NULL) prop.Destruct(&pInputName); @@ -657,9 +678,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } discardFunixn(); - nfunix = 1; + nfd = 1; bIgnoreTimestamp = 1; bUseFlowCtl = 0; + bUseCreds = 0; bCreateSockPath = DFLT_bCreateSockPath; return RS_RET_OK; @@ -679,10 +701,20 @@ CODEmodInit_QueryRegCFSLineHdlr dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); - /* initialize funixn[] array */ + /* init system log socket settings */ + listeners[0].flags = IGNDATE; + listeners[0].sockName = UCHAR_CONSTANT(_PATH_LOG); + listeners[0].hostName = NULL; + listeners[0].flowCtl = eFLOWCTL_NO_DELAY; + listeners[0].fd = -1; + listeners[0].bParseHost = 0; + listeners[0].bUseCreds = 0; + listeners[0].bCreateSockPath = 0; + + /* initialize socket names */ for(i = 1 ; i < MAXFUNIX ; ++i) { - funixn[i] = NULL; - funix[i] = -1; + listeners[i].sockName = NULL; + listeners[i].fd = -1; } CHKiRet(prop.Construct(&pLocalHostIP)); @@ -690,9 +722,9 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(prop.ConstructFinalize(pLocalHostIP)); /* now init listen socket zero, the local log socket */ - CHKiRet(prop.Construct(&(funixHName[0]))); - CHKiRet(prop.SetString(funixHName[0], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); - CHKiRet(prop.ConstructFinalize(funixHName[0])); + CHKiRet(prop.Construct(&(listeners[0].hostName))); + CHKiRet(prop.SetString(listeners[0].hostName, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.ConstructFinalize(listeners[0].hostName)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"omitlocallogging", 0, eCmdHdlrBinary, @@ -707,6 +739,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, NULL, &bCreateSockPath, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusespidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit From 3773aadcdd8008b97bb532f6d29fe29d17b06159 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 11:41:17 +0200 Subject: added support for SCM_CREDENTIALS to imuxsock (now fully working) --- plugins/imuxsock/imuxsock.c | 197 +++++++++++++++++++++----------------------- 1 file changed, 92 insertions(+), 105 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 493f11c4..df2b8efc 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -83,7 +83,7 @@ statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) /* structure to describe a specific listener */ -struct lstn_s { +typedef struct lstn_s { uchar *sockName; /* read-only after startup */ prop_t *hostName; /* host-name override - if set, use this instead of actual name */ int fd; /* read-only after startup */ @@ -91,9 +91,9 @@ struct lstn_s { int flowCtl; /* flow control settings for this socket */ sbool bParseHost; /* should parser parse host name? read-only after startup */ sbool bUseCreds; - sbool bCreateSockPath; /* auto-creation of socket directory? */ -}; -static struct lstn_s listeners[MAXFUNIX]; + sbool bCreatePath; /* auto-creation of socket directory? */ +} lstn_t; +static lstn_t listeners[MAXFUNIX]; static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ @@ -108,10 +108,11 @@ static int bOmitLocalLogging = 0; static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ -static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ -static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ -#define DFLT_bCreateSockPath 0 -static int bCreateSockPath = DFLT_bCreateSockPath; /* auto-create socket path? */ +static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +#define DFLT_bCreatePath 0 +static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ /* set the timestamp ignore / not ignore option for the system @@ -165,7 +166,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; - listeners[nfd].bCreateSockPath = bCreateSockPath; + listeners[nfd].bCreatePath = bCreatePath; listeners[nfd].sockName = pNewVal; listeners[nfd].bUseCreds = bUseCreds; nfd++; @@ -204,24 +205,25 @@ static rsRetVal discardFunixn(void) /* used to create a log socket if NOT passed in via systemd. */ static inline rsRetVal -createLogSocket(const char *path, int bCreatePath, int *fd) +createLogSocket(lstn_t *pLstn) { struct sockaddr_un sunx; DEFiRet; - unlink(path); + unlink((char*)pLstn->sockName); memset(&sunx, 0, sizeof(sunx)); sunx.sun_family = AF_UNIX; - if(bCreatePath) { - makeFileParentDirs((uchar*)path, strlen(path), 0755, -1, -1, 0); + if(pLstn->bCreatePath) { + makeFileParentDirs((uchar*)pLstn->sockName, ustrlen(pLstn->sockName), 0755, -1, -1, 0); } - strncpy(sunx.sun_path, path, sizeof(sunx.sun_path)); - *fd = socket(AF_UNIX, SOCK_DGRAM, 0); - if(*fd < 0 || bind(*fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || - chmod(path, 0666) < 0) { - errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", path); - dbgprintf("cannot create %s (%d).\n", path, errno); - close(*fd); + strncpy(sunx.sun_path, (char*)pLstn->sockName, sizeof(sunx.sun_path)); + pLstn->fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if(pLstn->fd < 0 || bind(pLstn->fd, (struct sockaddr *) &sunx, SUN_LEN(&sunx)) < 0 || + chmod((char*)pLstn->sockName, 0666) < 0) { + errmsg.LogError(errno, NO_ERRCODE, "connot create '%s'", pLstn->sockName); + dbgprintf("cannot create %s (%d).\n", pLstn->sockName, errno); + close(pLstn->fd); + pLstn->fd = -1; ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } finalize_it: @@ -230,16 +232,15 @@ finalize_it: static inline rsRetVal -openLogSocket(const char *path, int bCreatePath, int *pfd) +openLogSocket(lstn_t *pLstn) { DEFiRet; - int fd; int one; - if (path[0] == '\0') + if(pLstn->sockName[0] == '\0') return -1; - if (strcmp(path, _PATH_LOG) == 0) { + if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { int r; /* System log socket code. Check whether an FD was passed in from systemd. If @@ -257,8 +258,8 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) } if (r == 1) { - fd = SD_LISTEN_FDS_START; - r = sd_is_socket_unix(fd, SOCK_DGRAM, -1, _PATH_LOG, 0); + pLstn->fd = SD_LISTEN_FDS_START; + r = sd_is_socket_unix(pLstn->fd, SOCK_DGRAM, -1, _PATH_LOG, 0); if (r < 0) { errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); @@ -269,42 +270,39 @@ openLogSocket(const char *path, int bCreatePath, int *pfd) ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } } else { - CHKiRet(createLogSocket(path, bCreatePath, &fd)); + CHKiRet(createLogSocket(pLstn)); } } else { - CHKiRet(createLogSocket(path, bCreatePath, &fd)); + CHKiRet(createLogSocket(pLstn)); } - one = 1; - if(setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - if(setsockopt(fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + if(pLstn->bUseCreds) { + one = 1; + if(setsockopt(pLstn->fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_PASSCRED failed on '%s'", pLstn->sockName); + pLstn->bUseCreds = 0; + } + if(setsockopt(pLstn->fd, SOL_SOCKET, SCM_CREDENTIALS, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS failed on '%s'", pLstn->sockName); + pLstn->bUseCreds = 0; + } } + +#if 0 + // SO_TIMESTAMP currently does not work for an unknown reason. Any help is appreciated! one = 1; - if(setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); - close(fd); + if(setsockopt(pLstn->fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", pLstn->sockName); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } +#endif - - -static socklen_t two =sizeof(one); - if(getsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &one,&two) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", path); - close(fd); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); +finalize_it: + if(iRet != RS_RET_OK) { + close(pLstn->fd); + pLstn->fd = -1; } -dbgprintf("imuxsock: SO_TIMESTAMP is %d\n", one); - *pfd = fd; -finalize_it: RETiRet; } @@ -321,7 +319,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) if(cred == NULL) return; - lenPID = snprintf(bufPID, sizeof(bufPID), "[%u]:", (unsigned) cred->pid); + lenPID = snprintf(bufPID, sizeof(bufPID), "[%lu]:", (unsigned long) cred->pid); for(i = *lenTag ; i >= 0 && bufTAG[i] != '[' ; --i) /*JUST SKIP*/; @@ -342,7 +340,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) * can also mangle it if necessary. */ static inline rsRetVal -SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) +SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) { msg_t *pMsg; int lenMsg; @@ -356,7 +354,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) CHKiRet(msgConstruct(&pMsg)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); - MsgSetFlowControlType(pMsg, listeners[iSock].flowCtl); + MsgSetFlowControlType(pMsg, pLstn->flowCtl); // TODO: handle format errors parse = pRcv; @@ -372,17 +370,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, int iSock, struct ucred *cred) pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); -dbgprintf("imuxsock: submit: facil %d, sever %d\n", pMsg->iFacility, pMsg->iSeverity); parse++; lenMsg--; /* '>' */ -dbgprintf("imuxsock: submit: stage 2\n"); if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { dbgprintf("we have a problem, invalid timestamp in msg!\n"); } /* pull tag */ -dbgprintf("imuxsock: submit: stage 3\n"); i = 0; while(lenMsg > 0 && *parse != ' ' && i < CONF_TAG_MAXSIZE) { @@ -395,19 +390,13 @@ dbgprintf("imuxsock: submit: stage 3\n"); MsgSetMSGoffs(pMsg, lenRcv - lenMsg); - // TODO: here we need to mangle the raw message if we need to - // "fix up" the user pid. In the long term, it may make more sense - // to add this (somehow) to the message object (problem: at this stage, - // it is not fully available, eg no structured data). Alternatively, we - // may use a custom parser for our messages, but that doesn't play too well - // with the rest of the system. - if(listeners[iSock].bParseHost) { - pMsg->msgFlags = listeners[iSock].flags | PARSE_HOSTNAME; + if(pLstn->bParseHost) { + pMsg->msgFlags = pLstn->flags | PARSE_HOSTNAME; } else { - pMsg->msgFlags = listeners[iSock].flags; + pMsg->msgFlags = pLstn->flags; } - MsgSetRcvFrom(pMsg, listeners[iSock].hostName); + MsgSetRcvFrom(pMsg, pLstn->hostName); CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); CHKiRet(submitMsg(pMsg)); @@ -424,21 +413,20 @@ finalize_it: * of the socket which is to be processed. This eases access to the * growing number of properties. -- rgerhards, 2008-08-01 */ -static rsRetVal readSocket(int fd, int iSock) +static rsRetVal readSocket(lstn_t *pLstn) { DEFiRet; int iRcvd; int iMaxLine; struct msghdr msgh; struct iovec msgiov; - static int ratelimitErrmsg = 0; // TODO: atomic OPS struct cmsghdr *cm; struct ucred *cred; uchar bufRcv[4096+1]; - char aux[1024]; + char aux[128]; uchar *pRcv = NULL; /* receive buffer */ - assert(iSock >= 0); + assert(pLstn->fd >= 0); iMaxLine = glbl.GetMaxLine(); @@ -456,38 +444,34 @@ static rsRetVal readSocket(int fd, int iSock) memset(&msgh, 0, sizeof(msgh)); memset(&msgiov, 0, sizeof(msgiov)); - memset(&aux, 0, sizeof(aux)); + if(pLstn->bUseCreds) { + memset(&aux, 0, sizeof(aux)); + msgh.msg_control = aux; + msgh.msg_controllen = sizeof(aux); + } msgiov.iov_base = pRcv; msgiov.iov_len = iMaxLine; msgh.msg_iov = &msgiov; msgh.msg_iovlen = 1; - msgh.msg_control = aux; - msgh.msg_controllen = sizeof(aux); - iRcvd = recvmsg(fd, &msgh, MSG_DONTWAIT); -/* iRcvd = recv(fd, pRcv, iMaxLine, 0); - */ + iRcvd = recvmsg(pLstn->fd, &msgh, MSG_DONTWAIT); - dbgprintf("Message from UNIX socket: #%d\n", fd); - if (iRcvd > 0) { - ratelimitErrmsg = 0; - - -dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); - cred = NULL; - for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { -dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); - if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { - cred = (struct ucred*) CMSG_DATA(cm); - dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); - //break; - } - } -dbgprintf("XXX: post CM loop\n"); - - - - CHKiRet(SubmitMsg(pRcv, iRcvd, iSock, cred)); - } else if (iRcvd < 0 && errno != EINTR && ratelimitErrmsg++ < 100) { + dbgprintf("Message from UNIX socket: #%d\n", pLstn->fd); + if(iRcvd > 0) { + cred = NULL; + if(pLstn->bUseCreds) { + dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); + for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { + dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { + cred = (struct ucred*) CMSG_DATA(cm); + dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); + //break; + } + } + dbgprintf("XXX: post CM loop\n"); + } + CHKiRet(SubmitMsg(pRcv, iRcvd, pLstn, cred)); + } else if(iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("UNIX socket error: %d = %s.\n", errno, errStr); @@ -555,7 +539,7 @@ CODESTARTrunInput if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ if ((fd = listeners[i].fd) != -1 && FD_ISSET(fd, pReadfds)) { - readSocket(fd, i); + readSocket(&(listeners[i])); --nfds; /* indicate we have processed one */ } } @@ -585,12 +569,12 @@ CODESTARTwillRun # endif if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; + listeners[0].bUseCreds = bUseCredsSysSock; /* initialize and return if will run or not */ actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfd ; i++) { - if(openLogSocket((char*) listeners[i].sockName, listeners[i].bCreateSockPath, - &(listeners[i].fd)) == RS_RET_OK) { + if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } @@ -682,7 +666,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bIgnoreTimestamp = 1; bUseFlowCtl = 0; bUseCreds = 0; - bCreateSockPath = DFLT_bCreateSockPath; + bUseCredsSysSock = 0; + bCreatePath = DFLT_bCreatePath; return RS_RET_OK; } @@ -709,7 +694,7 @@ CODEmodInit_QueryRegCFSLineHdlr listeners[0].fd = -1; listeners[0].bParseHost = 0; listeners[0].bUseCreds = 0; - listeners[0].bCreateSockPath = 0; + listeners[0].bCreatePath = 0; /* initialize socket names */ for(i = 1 ; i < MAXFUNIX ; ++i) { @@ -738,9 +723,11 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketflowcontrol", 0, eCmdHdlrBinary, NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, - NULL, &bCreateSockPath, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusespidfromsystem", 0, eCmdHdlrBinary, + NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit From b5da6352830e9841dd367b8490d79461adb5cb22 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Sep 2010 15:06:30 +0200 Subject: first shot at imuxsock ratelimiting works, but at a global level, need to go down to pid or cgroup --- plugins/imuxsock/imuxsock.c | 98 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index df2b8efc..ccd31b44 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -81,6 +81,15 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) +STATSCOUNTER_DEF(ctrLostRatelimit, mutCtrLostRatelimit) + +struct rs_ratelimit_state { + unsigned short interval; + unsigned short burst; + unsigned done; + unsigned missed; + time_t begin; +} ratelimiter; /* structure to describe a specific listener */ typedef struct lstn_s { @@ -113,6 +122,70 @@ static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TA static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ +#define DFLT_ratelimitInterval 2 +static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ +#define DFLT_ratelimitBurst 200 +static int ratelimitBurst = DFLT_ratelimitBurst; /* max nbr of messages in interval */ + + + +static void +initRatelimitState(struct rs_ratelimit_state *rs, unsigned short interval, unsigned short burst) +{ + rs->interval = interval; + rs->burst = burst; + rs->done = 0; + rs->missed = 0; + rs->begin = 0; +} + + +/* ratelimiting support, modelled after the linux kernel + * returns 1 if message is within rate limit and shall be + * processed, 0 otherwise. + * This implementation is NOT THREAD-SAFE and must not + * be called concurrently. + */ +static inline int +withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) +{ + int ret; + uchar msgbuf[1024]; + + if(rs->interval == 0) { + ret = 1; + goto finalize_it; + } + + assert(rs->burst != 0); + + if(rs->begin == 0) + rs->begin = tt; + + /* resume if we go out of out time window */ + if(tt > rs->begin + rs->interval) { + if(rs->missed) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "imuxsock lost %u messages due to rate-limiting", rs->missed); + logmsgInternal(0, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + rs->missed = 0; + } + rs->begin = 0; + rs->done = 0; + } + + /* do actual limit check */ + if(rs->burst > rs->done) { + rs->done++; + ret = 1; + } else { + rs->missed++; + ret = 0; + } + +finalize_it: + return ret; +} /* set the timestamp ignore / not ignore option for the system @@ -348,14 +421,23 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) uchar *parse; int pri; uchar bufParseTAG[CONF_TAG_MAXSIZE]; + struct syslogTime st; + time_t tt; DEFiRet; + datetime.getCurrTime(&st, &tt); + if(!withinRatelimit(&ratelimiter, tt)) { + STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); + FINALIZE; + } + /* we now create our own message object and submit it to the queue */ - CHKiRet(msgConstruct(&pMsg)); + CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); + // TODO: handle format errors parse = pRcv; lenMsg = lenRcv; @@ -504,6 +586,8 @@ CODESTARTrunInput * signalled to do so. This, however, is handled by the framework, * right into the sleep below. */ + initRatelimitState(&ratelimiter, ratelimitInterval, ratelimitBurst); + while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -668,6 +752,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bUseCreds = 0; bUseCredsSysSock = 0; bCreatePath = DFLT_bCreatePath; + ratelimitInterval = DFLT_ratelimitInterval; + ratelimitBurst = DFLT_ratelimitBurst; return RS_RET_OK; } @@ -726,10 +812,12 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitinterval", 0, eCmdHdlrInt, + NULL, &ratelimitInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitburst", 0, eCmdHdlrInt, + NULL, &ratelimitBurst, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); /* the following one is a (dirty) trick: the system log socket is not added via @@ -742,12 +830,16 @@ CODEmodInit_QueryRegCFSLineHdlr setSystemLogTimestampIgnore, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, + NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); /* support statistics gathering */ CHKiRet(statsobj.Construct(&modStats)); CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), ctrType_IntCtr, &ctrSubmit)); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("lost.ratelimit"), + ctrType_IntCtr, &ctrLostRatelimit)); CHKiRet(statsobj.ConstructFinalize(modStats)); ENDmodInit -- cgit From d748c68c09ae7d4f8c22c4245e5509019c59511c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 11:32:48 +0200 Subject: added some generic hashtable code by Christopher Clark found at http://www.cl.cam.ac.uk/~cwc22/hashtable/ --- plugins/imuxsock/Makefile.am | 2 +- plugins/imuxsock/imuxsock.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imuxsock/Makefile.am b/plugins/imuxsock/Makefile.am index 28f9f9e3..34a0ad9a 100644 --- a/plugins/imuxsock/Makefile.am +++ b/plugins/imuxsock/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = imuxsock.la imuxsock_la_SOURCES = imuxsock.c -imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +imuxsock_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -I../../runtime/hashtable -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imuxsock_la_LDFLAGS = -module -avoid-version imuxsock_la_LIBADD = $(RSRT_LIBS) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index ccd31b44..b53bb379 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -52,6 +52,7 @@ #include "sd-daemon.h" #include "statsobj.h" #include "datetime.h" +#include "hashtable.h" MODULE_TYPE_INPUT -- cgit From 01a8807174d91a7936345d1172a87f98bbba61c4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 12:13:19 +0200 Subject: imuxsock: changed to per-pid ratelimiting --- plugins/imuxsock/imuxsock.c | 83 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 6 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index b53bb379..728456b6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-20 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -83,6 +83,7 @@ DEFobjCurrIf(statsobj) statsobj_t *modStats; STATSCOUNTER_DEF(ctrSubmit, mutCtrSubmit) STATSCOUNTER_DEF(ctrLostRatelimit, mutCtrLostRatelimit) +STATSCOUNTER_DEF(ctrNumRatelimiters, mutCtrNumRatelimiters) struct rs_ratelimit_state { unsigned short interval; @@ -90,7 +91,30 @@ struct rs_ratelimit_state { unsigned done; unsigned missed; time_t begin; -} ratelimiter; +}; +typedef struct rs_ratelimit_state rs_ratelimit_state_t; + +/* support for per-process ratelimiting */ +static struct hashtable *hashtab; + +/* a very simple "hash function" for process IDs - we simply use the + * pid itself: it is quite expected that all pids may log some time, but + * from a collision point of view it is likely that long-running daemons + * start early and so will stay right in the top spots of the + * collision list. + */ +static unsigned int +hash_from_key_fn(void *k) +{ + return((unsigned) *((pid_t*) k)); +} + +static int +key_equals_fn(void *key1, void *key2) +{ + return *((pid_t*) key1) == *((pid_t*) key2); +} + /* structure to describe a specific listener */ typedef struct lstn_s { @@ -381,6 +405,43 @@ finalize_it: } +/* find ratelimiter to use for this message. Currently, we use the + * pid, but may change to cgroup later (probably via a config switch). + * Returns NULL if not found. + */ +static inline rsRetVal +findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) +{ + rs_ratelimit_state_t *rl; + int r; + pid_t *keybuf; + DEFiRet; + + if(cred == NULL) + FINALIZE; + + rl = hashtable_search(hashtab, &cred->pid); + if(rl == NULL) { + /* we need to add a new ratelimiter, process not seen before! */ + dbgprintf("imuxsock: no ratelimiter for pid %lu, creating one\n", + (unsigned long) cred->pid); + STATSCOUNTER_INC(ctrNumRatelimiters, mutCtrNumRatelimiters); + CHKmalloc(rl = malloc(sizeof(rs_ratelimit_state_t))); + CHKmalloc(keybuf = malloc(sizeof(pid_t))); + *keybuf = cred->pid; + initRatelimitState(rl, ratelimitInterval, ratelimitBurst); + r = hashtable_insert(hashtab, keybuf, rl); + if(r == 0) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + + *prl = rl; + +finalize_it: + RETiRet; +} + + /* patch correct pid into tag. bufTAG MUST be CONF_TAG_MAXSIZE long! */ static inline void @@ -424,10 +485,13 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) uchar bufParseTAG[CONF_TAG_MAXSIZE]; struct syslogTime st; time_t tt; + rs_ratelimit_state_t *ratelimiter; DEFiRet; + findRatelimiter(cred, &ratelimiter); /* ignore error, better so than others... */ + datetime.getCurrTime(&st, &tt); - if(!withinRatelimit(&ratelimiter, tt)) { + if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); FINALIZE; } @@ -587,8 +651,6 @@ CODESTARTrunInput * signalled to do so. This, however, is handled by the framework, * right into the sleep below. */ - initRatelimitState(&ratelimiter, ratelimitInterval, ratelimitBurst); - while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -675,6 +737,8 @@ CODESTARTwillRun CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); + CHKmalloc(hashtab = create_hashtable(1000, hash_from_key_fn, key_equals_fn)); + finalize_it: ENDwillRun @@ -706,6 +770,11 @@ CODESTARTafterRun if(pInputName != NULL) prop.Destruct(&pInputName); + + if(hashtab != NULL) { + hashtable_destroy(hashtab, 1); /* 1 => free all values automatically */ + hashtab = NULL; + } ENDafterRun @@ -839,8 +908,10 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("submitted"), ctrType_IntCtr, &ctrSubmit)); - CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("lost.ratelimit"), + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("ratelimit.discarded"), ctrType_IntCtr, &ctrLostRatelimit)); + CHKiRet(statsobj.AddCounter(modStats, UCHAR_CONSTANT("ratelimit.numratelimiters"), + ctrType_IntCtr, &ctrNumRatelimiters)); CHKiRet(statsobj.ConstructFinalize(modStats)); ENDmodInit -- cgit From 054d2ccdd6044f94823f8facbda935cb70646333 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 17:26:28 +0200 Subject: imuxsock: added per-socket hash tables/rate limiters & severity filter rate limiting now applies only to messages with a given severity or above. By default, emergency messages are NOT rate-limited. --- plugins/imuxsock/imuxsock.c | 159 +++++++++++++++++++++++++++++--------------- 1 file changed, 104 insertions(+), 55 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 728456b6..d500fc54 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -57,7 +57,7 @@ MODULE_TYPE_INPUT /* defines */ -#define MAXFUNIX 20 +#define MAXFUNIX 50 #ifndef _PATH_LOG #ifdef BSD #define _PATH_LOG "/var/run/log" @@ -94,8 +94,6 @@ struct rs_ratelimit_state { }; typedef struct rs_ratelimit_state rs_ratelimit_state_t; -/* support for per-process ratelimiting */ -static struct hashtable *hashtab; /* a very simple "hash function" for process IDs - we simply use the * pid itself: it is quite expected that all pids may log some time, but @@ -118,14 +116,19 @@ key_equals_fn(void *key1, void *key2) /* structure to describe a specific listener */ typedef struct lstn_s { - uchar *sockName; /* read-only after startup */ - prop_t *hostName; /* host-name override - if set, use this instead of actual name */ - int fd; /* read-only after startup */ - int flags; /* should parser parse host name? read-only after startup */ - int flowCtl; /* flow control settings for this socket */ - sbool bParseHost; /* should parser parse host name? read-only after startup */ - sbool bUseCreds; - sbool bCreatePath; /* auto-creation of socket directory? */ + uchar *sockName; /* read-only after startup */ + prop_t *hostName; /* host-name override - if set, use this instead of actual name */ + int fd; /* read-only after startup */ + int flags; /* should parser parse host name? read-only after startup */ + int flowCtl; /* flow control settings for this socket */ + int ratelimitInterval; + int ratelimitBurst; + intTiny ratelimitSev; /* severity level (and below) for which rate-limiting shall apply */ + struct hashtable *ht; /* our hashtable for rate-limiting */ + sbool bParseHost; /* should parser parse host name? read-only after startup */ + sbool bCreatePath; /* auto-creation of socket directory? */ + sbool bUseCreds; /* pull original creator credentials from socket */ + sbool bWritePid; /* write original PID into tag */ } lstn_t; static lstn_t listeners[MAXFUNIX]; @@ -142,15 +145,20 @@ static int bOmitLocalLogging = 0; static uchar *pLogSockName = NULL; static uchar *pLogHostName = NULL; /* host name to use with this socket */ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is used! */ -static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ -static int bUseCreds = 0; /* use credentials from recvmsg() and fixup PID in TAG */ -static int bUseCredsSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ +static int bWritePid = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bWritePidSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ #define DFLT_ratelimitInterval 2 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ +static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 static int ratelimitBurst = DFLT_ratelimitBurst; /* max nbr of messages in interval */ +static int ratelimitBurstSysSock = DFLT_ratelimitBurst; /* max nbr of messages in interval */ +#define DFLT_ratelimitSeverity 1 /* do not rate-limit emergency messages */ +static int ratelimitSeverity = DFLT_ratelimitSeverity; +static int ratelimitSeveritySysSock = DFLT_ratelimitSeverity; @@ -192,7 +200,7 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) if(rs->missed) { snprintf((char*)msgbuf, sizeof(msgbuf), "imuxsock lost %u messages due to rate-limiting", rs->missed); - logmsgInternal(0, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); rs->missed = 0; } rs->begin = 0; @@ -262,11 +270,23 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) pLogHostName = NULL; } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); + if(ratelimitInterval > 0) { + if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + /* in this case, we simply turn of rate-limiting */ + dbgprintf("imuxsock: turning off rate limiting because we could not " + "create hash table\n"); + ratelimitInterval = 0; + } + } + listeners[nfd].ratelimitInterval = ratelimitInterval; + listeners[nfd].ratelimitBurst = ratelimitBurst; + listeners[nfd].ratelimitSev = ratelimitSeverity; listeners[nfd].flowCtl = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; listeners[nfd].bCreatePath = bCreatePath; listeners[nfd].sockName = pNewVal; - listeners[nfd].bUseCreds = bUseCreds; + listeners[nfd].bUseCreds = (bWritePid || ratelimitInterval) ? 1 : 0; + listeners[nfd].bWritePid = bWritePid; nfd++; } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", @@ -278,11 +298,10 @@ finalize_it: } -/* free the sockName[] socket names - needed as cleanup on several places - * note that nfd is NOT reset! sockName[0] is never freed, as it comes from +/* discard all log sockets except for "socket" 0. Data for it comes from * the constant memory pool - and if not, it is freeed via some other pointer. */ -static rsRetVal discardFunixn(void) +static rsRetVal discardLogSockets(void) { int i; @@ -294,6 +313,9 @@ static rsRetVal discardFunixn(void) if(listeners[i].hostName != NULL) { prop.Destruct(&(listeners[i].hostName)); } + if(listeners[i].ht != NULL) { + hashtable_destroy(listeners[i].ht, 1); /* 1 => free all values automatically */ + } } return RS_RET_OK; @@ -410,7 +432,7 @@ finalize_it: * Returns NULL if not found. */ static inline rsRetVal -findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) +findRatelimiter(lstn_t *pLstn, struct ucred *cred, rs_ratelimit_state_t **prl) { rs_ratelimit_state_t *rl; int r; @@ -420,7 +442,7 @@ findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) if(cred == NULL) FINALIZE; - rl = hashtable_search(hashtab, &cred->pid); + rl = hashtable_search(pLstn->ht, &cred->pid); if(rl == NULL) { /* we need to add a new ratelimiter, process not seen before! */ dbgprintf("imuxsock: no ratelimiter for pid %lu, creating one\n", @@ -429,8 +451,8 @@ findRatelimiter(struct ucred *cred, rs_ratelimit_state_t **prl) CHKmalloc(rl = malloc(sizeof(rs_ratelimit_state_t))); CHKmalloc(keybuf = malloc(sizeof(pid_t))); *keybuf = cred->pid; - initRatelimitState(rl, ratelimitInterval, ratelimitBurst); - r = hashtable_insert(hashtab, keybuf, rl); + initRatelimitState(rl, ratelimitInterval, pLstn->ratelimitBurst); + r = hashtable_insert(pLstn->ht, keybuf, rl); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -482,13 +504,33 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) int i; uchar *parse; int pri; + int facil; + int sever; uchar bufParseTAG[CONF_TAG_MAXSIZE]; struct syslogTime st; time_t tt; - rs_ratelimit_state_t *ratelimiter; + rs_ratelimit_state_t *ratelimiter = NULL; DEFiRet; - findRatelimiter(cred, &ratelimiter); /* ignore error, better so than others... */ +// TODO: handle format errors?? + /* we need to parse the pri first, because we need the severity for + * rate-limiting as well. + */ + parse = pRcv; + lenMsg = lenRcv; + + parse++; lenMsg--; /* '<' */ + pri = 0; + while(lenMsg && isdigit(*parse)) { + pri = pri * 10 + *parse - '0'; + ++parse; + --lenMsg; + } + facil = LOG_FAC(pri); + sever = LOG_PRI(pri); + + if(sever >= pLstn->ratelimitSev) + findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ datetime.getCurrTime(&st, &tt); if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { @@ -502,26 +544,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); - -// TODO: handle format errors - parse = pRcv; - lenMsg = lenRcv; - - parse++; lenMsg--; /* '<' */ - pri = 0; - while(lenMsg && isdigit(*parse)) { - pri = pri * 10 + *parse - '0'; - ++parse; - --lenMsg; - } - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); + pMsg->iFacility = facil; + pMsg->iSeverity = sever; MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); parse++; lenMsg--; /* '>' */ if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { - dbgprintf("we have a problem, invalid timestamp in msg!\n"); + DBGPRINTF("we have a problem, invalid timestamp in msg!\n"); } /* pull tag */ @@ -532,7 +562,8 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) --lenMsg; } bufParseTAG[i] = '\0'; /* terminate string */ - fixPID(bufParseTAG, &i, cred); + if(pLstn->bWritePid) + fixPID(bufParseTAG, &i, cred); MsgSetTAG(pMsg, bufParseTAG, i); MsgSetMSGoffs(pMsg, lenRcv - lenMsg); @@ -716,7 +747,19 @@ CODESTARTwillRun # endif if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; - listeners[0].bUseCreds = bUseCredsSysSock; + if(ratelimitIntervalSysSock > 0) { + if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + /* in this case, we simply turn of rate-limiting */ + dbgprintf("imuxsock: turning off rate limiting because we could not " + "create hash table\n"); + ratelimitIntervalSysSock = 0; + } + } + listeners[0].ratelimitInterval = ratelimitIntervalSysSock; + listeners[0].ratelimitBurst = ratelimitBurstSysSock; + listeners[0].ratelimitSev = ratelimitSeveritySysSock; + listeners[0].bUseCreds = (bWritePidSysSock || ratelimitIntervalSysSock) ? 1 : 0; + listeners[0].bWritePid = bWritePidSysSock; /* initialize and return if will run or not */ actSocks = 0; @@ -724,7 +767,7 @@ CODESTARTwillRun if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); - } + } } if(actSocks == 0) { @@ -737,8 +780,6 @@ CODESTARTwillRun CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imuxsock"), sizeof("imuxsock") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); - CHKmalloc(hashtab = create_hashtable(1000, hash_from_key_fn, key_equals_fn)); - finalize_it: ENDwillRun @@ -765,16 +806,12 @@ CODESTARTafterRun free(pLogSockName); free(pLogHostName); - discardFunixn(); + discardLogSockets(); nfd = 1; if(pInputName != NULL) prop.Destruct(&pInputName); - if(hashtab != NULL) { - hashtable_destroy(hashtab, 1); /* 1 => free all values automatically */ - hashtab = NULL; - } ENDafterRun @@ -815,15 +852,19 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a pLogHostName = NULL; } - discardFunixn(); + discardLogSockets(); nfd = 1; bIgnoreTimestamp = 1; bUseFlowCtl = 0; - bUseCreds = 0; - bUseCredsSysSock = 0; + bWritePid = 0; + bWritePidSysSock = 0; bCreatePath = DFLT_bCreatePath; ratelimitInterval = DFLT_ratelimitInterval; + ratelimitIntervalSysSock = DFLT_ratelimitInterval; ratelimitBurst = DFLT_ratelimitBurst; + ratelimitBurstSysSock = DFLT_ratelimitBurst; + ratelimitSeverity = DFLT_ratelimitSeverity; + ratelimitSeveritySysSock = DFLT_ratelimitSeverity; return RS_RET_OK; } @@ -881,13 +922,15 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCreds, STD_LOADABLE_MODULE_ID)); + NULL, &bWritePid, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitinterval", 0, eCmdHdlrInt, NULL, &ratelimitInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitburst", 0, eCmdHdlrInt, NULL, &ratelimitBurst, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitseverity", 0, eCmdHdlrInt, + NULL, &ratelimitSeverity, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); /* the following one is a (dirty) trick: the system log socket is not added via @@ -901,7 +944,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, - NULL, &bUseCredsSysSock, STD_LOADABLE_MODULE_ID)); + NULL, &bWritePidSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitinterval", 0, eCmdHdlrInt, + NULL, &ratelimitIntervalSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitburst", 0, eCmdHdlrInt, + NULL, &ratelimitBurstSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitseverity", 0, eCmdHdlrInt, + NULL, &ratelimitSeveritySysSock, STD_LOADABLE_MODULE_ID)); /* support statistics gathering */ CHKiRet(statsobj.Construct(&modStats)); -- cgit From f4d6418102033c22b0ae71d496e993b00773b5c4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 17:52:15 +0200 Subject: doc/imuxsock: added new options to doc --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index d500fc54..b4ee8db0 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -150,7 +150,7 @@ static int bWritePid = 0; /* use credentials from recvmsg() and fixup PID in TA static int bWritePidSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ -#define DFLT_ratelimitInterval 2 +#define DFLT_ratelimitInterval 5 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 -- cgit From 33888b84e7dc6ec4341e066c68c7077cade24a7b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 29 Sep 2010 14:31:44 +0200 Subject: imuxsock: indicate when rate limiting begins, include pid in rl messages also improved test tool --- plugins/imuxsock/imuxsock.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index b4ee8db0..24bcebb7 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -180,7 +180,7 @@ initRatelimitState(struct rs_ratelimit_state *rs, unsigned short interval, unsig * be called concurrently. */ static inline int -withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) +withinRatelimit(struct rs_ratelimit_state *rs, time_t tt, pid_t pid) { int ret; uchar msgbuf[1024]; @@ -199,7 +199,8 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) if(tt > rs->begin + rs->interval) { if(rs->missed) { snprintf((char*)msgbuf, sizeof(msgbuf), - "imuxsock lost %u messages due to rate-limiting", rs->missed); + "imuxsock lost %u messages from pid %lu due to rate-limiting", + rs->missed, (unsigned long) pid); logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); rs->missed = 0; } @@ -212,6 +213,12 @@ withinRatelimit(struct rs_ratelimit_state *rs, time_t tt) rs->done++; ret = 1; } else { + if(rs->missed == 0) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "imuxsock begins to drop messages from pid %lu due to rate-limiting", + (unsigned long) pid); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + } rs->missed++; ret = 0; } @@ -533,7 +540,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ datetime.getCurrTime(&st, &tt); - if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt)) { + if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt, cred->pid)) { STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); FINALIZE; } -- cgit From 11617d8ab5192c12e3b33cd9c08ac32f1d334a85 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 30 Sep 2010 14:17:34 +0200 Subject: omhdfs: first shot at this new module (very rough PoC code) --- plugins/omhdfs/Makefile.am | 6 + plugins/omhdfs/omhdfs.c | 296 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 plugins/omhdfs/Makefile.am create mode 100644 plugins/omhdfs/omhdfs.c (limited to 'plugins') diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am new file mode 100644 index 00000000..329e6816 --- /dev/null +++ b/plugins/omhdfs/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = omhdfs.la + +omhdfs_la_SOURCES = omhdfs.c +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +omhdfs_la_LDFLAGS = -module -avoid-version +omhdfs_la_LIBADD = diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c new file mode 100644 index 00000000..42e5dc8a --- /dev/null +++ b/plugins/omhdfs/omhdfs.c @@ -0,0 +1,296 @@ +/* omhdfs.c + * This is the implementation of the build-in file output module. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +/* this is kind of a hack, if defined, it instructs omhdfs to use + * the regular (non-hdfs) file system calls. This eases development + * (and hopefully troubleshooting) especially in cases when no + * hdfs environment is available. + */ +#define USE_REGULAR_FS 1 + +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "conf.h" +#include "cfsysline.h" +#include "module-template.h" +#include "unicode-helper.h" +#ifndef USE_REGULAR_FS +# include "hdfs.h" +#endif + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* globals for default values */ +static uchar *fileName = NULL; +/* end globals for default values */ + +typedef struct _instanceData { + uchar *fileName; +# ifdef USE_REGULAR_FS + short fd; /* file descriptor for (current) file */ +# else + hdfsFS fs; + hdfsFile fd; +# endif +} instanceData; + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + printf("omhdfs: file:%s", pData->fileName); + if (pData->fd == -1) + printf(" (unused)"); +ENDdbgPrintInstInfo + + + +#if 0 +static void prepareFile(instanceData *pData, uchar *newFileName) +{ + if(access((char*)newFileName, F_OK) == 0) { + /* file already exists */ + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fCreateMode); + } else { + pData->fd = -1; + /* file does not exist, create it (and eventually parent directories */ + if(pData->bCreateDirs) { + /* we fist need to create parent dirs if they are missing + * We do not report any errors here ourselfs but let the code + * fall through to error handler below. + */ + if(makeFileParentDirs(newFileName, strlen((char*)newFileName), + pData->fDirCreateMode, pData->dirUID, + pData->dirGID, pData->bFailOnChown) != 0) { + return; /* we give up */ + } + } + /* no matter if we needed to create directories or not, we now try to create + * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) + */ + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fCreateMode); + if(pData->fd != -1) { + /* check and set uid/gid */ + if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { + /* we need to set owner/group */ + if(fchown(pData->fd, pData->fileUID, + pData->fileGID) != 0) { + if(pData->bFailOnChown) { + int eSave = errno; + close(pData->fd); + pData->fd = -1; + errno = eSave; + } + /* we will silently ignore the chown() failure + * if configured to do so. + */ + } + } + } + } +} +#endif + +static void prepareFile(instanceData *pData, uchar *newFileName) +{ +# if USE_REGULAR_FS + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); +# else + pData->fs = hdfsConnect("default", 0); + pData->fd = hdfsOpenFile(fs, newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + if(!pData->fd) { + dbgprintf(stderr, "Failed to open %s for writing!\n", newFileName); + // TODO: suspend/error report + } + +# endif +} + +static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +{ + size_t lenWrite; + DEFiRet; + +# if USE_REGULAR_FS + if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { + int e = errno; + dbgprintf("omhdfs write error!\n"); + + /* If the filesystem is filled up, just ignore + * it for now and continue writing when possible + */ + //if(pData->fileType == eTypeFILE && e == ENOSPC) + //return RS_RET_OK; + + //(void) close(pData->fd); + iRet = RS_RET_DISABLE_ACTION; + errno = e; + //logerror((char*) pData->f_fname); + } +# else + lenWrite = strlen(char*ppString[0]); + tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); + if(num_written_bytes != lenWrite) { + dbgprintf("Failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, + lenWrite, num_written_bytes); + // TODO: suspend/error report + } +# endif +else { dbgprintf("omhdfs has successfully written to file\n"); } + + RETiRet; +} + + +BEGINcreateInstance +CODESTARTcreateInstance + pData->fd = -1; +ENDcreateInstance + + +BEGINfreeInstance +CODESTARTfreeInstance +# ifdef USE_REGULAR_FS + if(pData->fd != -1) + close(pData->fd); +# else + hdfsCloseFile(pData->fs, pData->fd); +# endif +ENDfreeInstance + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + dbgprintf(" (%s)\n", pData->fileName); + iRet = writeFile(ppString, iMsgOpts, pData); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct + + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omhdfs:", sizeof(":omhdfs:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* rgerhards 2004-11-17: from now, we need to have different + * processing, because after the first comma, the template name + * to use is specified. So we need to scan for the first coma first + * and then look at the rest of the line. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); + //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); + + // TODO: check for NULL filename + CHKmalloc(pData->fileName = ustrdup(fileName)); + + prepareFile(pData, pData->fileName); + +#if 0 + if ( pData->fd < 0 ){ + pData->fd = -1; + dbgprintf("Error opening log file: %s\n", pData->f_fname); + logerror((char*) pData->f_fname); + } +#endif +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +/* Reset config variables for this module to default values. + * rgerhards, 2007-07-17 + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ +/* + fileUID = -1; + fileGID = -1; + dirUID = -1; + dirGID = -1; + bFailOnChown = 1; + iDynaFileCacheSize = 10; + fCreateMode = 0644; + fDirCreateMode = 0700; + bCreateDirs = 1; +*/ + return RS_RET_OK; +} + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit -- cgit From 7ac7ad166b82034eea4a37c1937ca5ddd618ec45 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 30 Sep 2010 13:55:26 +0000 Subject: omhdfs: added "real" libhdfs code, now actually works on hdfs very crude implementation, but probably good enough to gather some early performance data and experience with the module. No real error handling done, if something breaks, the whole thing will be blown up ;) --- plugins/omhdfs/Makefile.am | 4 +-- plugins/omhdfs/omhdfs.c | 62 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 17 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 329e6816..23fcf75a 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -omhdfs_la_LDFLAGS = -module -avoid-version +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux +omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64 -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64/server -ljava -ljvm -lverify omhdfs_la_LIBADD = diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 42e5dc8a..fcdff6e5 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -28,7 +28,7 @@ * (and hopefully troubleshooting) especially in cases when no * hdfs environment is available. */ -#define USE_REGULAR_FS 1 +//#define USE_REGULAR_FS 1 #include "config.h" #include "rsyslog.h" @@ -62,6 +62,8 @@ DEF_OMOD_STATIC_DATA /* globals for default values */ static uchar *fileName = NULL; +static uchar *hdfsHost = NULL; +int hdfsPort = 0; /* end globals for default values */ typedef struct _instanceData { @@ -71,6 +73,8 @@ typedef struct _instanceData { # else hdfsFS fs; hdfsFile fd; + const char *hdfsHost; + tPort hdfsPort; # endif } instanceData; @@ -85,8 +89,8 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo printf("omhdfs: file:%s", pData->fileName); - if (pData->fd == -1) - printf(" (unused)"); + //if (pData->fd == -1) + //printf(" (unused)"); ENDdbgPrintInstInfo @@ -139,22 +143,41 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } #endif -static void prepareFile(instanceData *pData, uchar *newFileName) +static void +prepareFile(instanceData *pData, uchar *newFileName) { # if USE_REGULAR_FS pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); # else - pData->fs = hdfsConnect("default", 0); - pData->fd = hdfsOpenFile(fs, newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + dbgprintf("omhdfs: try to connect to host '%s' at port %d\n", + pData->hdfsHost, pData->hdfsPort); + pData->fs = hdfsConnect(pData->hdfsHost, pData->hdfsPort); + if(pData->fs == NULL) { + dbgprintf("omhdfs: error can not connect to hdfs\n"); + } + pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_APPEND, 0, 0, 0); + if(pData->fd == NULL) { + /* maybe the file does not exist, so we try to create it now. + * Note that we can not use hdfsExists() because of a deficit in + * it: https://issues.apache.org/jira/browse/HDFS-1154 + * As of my testing, libhdfs at least seems to return ENOENT if + * the file does not exist. + */ + if(errno == ENOENT) { + dbgprintf("omhdfs: ENOENT trying to append to '%s', now trying create\n", + newFileName); + pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + } + } if(!pData->fd) { - dbgprintf(stderr, "Failed to open %s for writing!\n", newFileName); + dbgprintf("omhdfs: failed to open %s for writing!\n", newFileName); // TODO: suspend/error report } # endif } -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal writeFile(uchar **ppString, instanceData *pData) { size_t lenWrite; DEFiRet; @@ -176,15 +199,14 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa //logerror((char*) pData->f_fname); } # else - lenWrite = strlen(char*ppString[0]); + lenWrite = strlen((char*) ppString[0]); tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); - if(num_written_bytes != lenWrite) { - dbgprintf("Failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, - lenWrite, num_written_bytes); + if((unsigned) num_written_bytes != lenWrite) { + dbgprintf("omhdfs: failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, + lenWrite, (unsigned long) num_written_bytes); // TODO: suspend/error report } # endif -else { dbgprintf("omhdfs has successfully written to file\n"); } RETiRet; } @@ -192,7 +214,7 @@ else { dbgprintf("omhdfs has successfully written to file\n"); } BEGINcreateInstance CODESTARTcreateInstance - pData->fd = -1; + //pData->fd = -1; ENDcreateInstance @@ -214,7 +236,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction dbgprintf(" (%s)\n", pData->fileName); - iRet = writeFile(ppString, iMsgOpts, pData); + iRet = writeFile(ppString, pData); ENDdoAction @@ -241,6 +263,12 @@ CODESTARTparseSelectorAct // TODO: check for NULL filename CHKmalloc(pData->fileName = ustrdup(fileName)); + if(hdfsHost == NULL) { + pData->hdfsHost = "default"; + } else { + CHKmalloc(pData->hdfsHost = strdup((char*)hdfsHost)); + } + pData->hdfsPort = hdfsPort; prepareFile(pData, pData->fileName); @@ -271,6 +299,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a fDirCreateMode = 0700; bCreateDirs = 1; */ + hdfsHost = NULL; + hdfsPort = 0; return RS_RET_OK; } @@ -291,6 +321,8 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr ENDmodInit -- cgit From 1d609fc5ab9b5d1c89d5d7d8f321c4e8493b4437 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 10:27:44 +0200 Subject: omhdfs: some cleanup of build system but still pretty ugly. Any java folks out there to help clean it up? --- plugins/omhdfs/Makefile.am | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 23fcf75a..2e7ef8ea 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,6 +1,7 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux -omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64 -L/usr/lib/jvm/java-6-sun-1.6.0.20/jre/lib/amd64/server -ljava -ljvm -lverify +#omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux +omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(JAVA_INCLUDES) +omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs $(JAVA_LIBS) omhdfs_la_LIBADD = -- cgit From d8a1489f545179591abced679ba24831d85ca224 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 11:29:50 +0200 Subject: omhdfs: cleanup and lots of improvement now things look much better, also done some prep in order to support a file cache (we need this for multiple selectors writing to the same file). --- plugins/omhdfs/omhdfs.c | 247 +++++++++++++++++++++++++++++------------------- 1 file changed, 150 insertions(+), 97 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index fcdff6e5..71080585 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -23,13 +23,6 @@ * A copy of the GPL can be found in the file "COPYING" in this distribution. */ -/* this is kind of a hack, if defined, it instructs omhdfs to use - * the regular (non-hdfs) file system calls. This eases development - * (and hopefully troubleshooting) especially in cases when no - * hdfs environment is available. - */ -//#define USE_REGULAR_FS 1 - #include "config.h" #include "rsyslog.h" #include @@ -42,6 +35,8 @@ #include #include #include +#include +#include #include "syslogd-types.h" #include "srUtils.h" @@ -50,15 +45,14 @@ #include "cfsysline.h" #include "module-template.h" #include "unicode-helper.h" -#ifndef USE_REGULAR_FS -# include "hdfs.h" -#endif +#include "errmsg.h" MODULE_TYPE_OUTPUT /* internal structures */ DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) /* globals for default values */ static uchar *fileName = NULL; @@ -66,18 +60,23 @@ static uchar *hdfsHost = NULL; int hdfsPort = 0; /* end globals for default values */ -typedef struct _instanceData { - uchar *fileName; -# ifdef USE_REGULAR_FS - short fd; /* file descriptor for (current) file */ -# else +typedef struct { + uchar *name; hdfsFS fs; - hdfsFile fd; + hdfsFile fh; const char *hdfsHost; tPort hdfsPort; -# endif + int nUsers; + pthread_mutex_t mut; +} file_t; + + +typedef struct _instanceData { + file_t *pFile; } instanceData; +/* forward definitions (down here, need data types) */ +static inline rsRetVal fileClose(file_t *pFile); BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -88,9 +87,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - printf("omhdfs: file:%s", pData->fileName); - //if (pData->fd == -1) - //printf(" (unused)"); + printf("omhdfs: file:%s", pData->pFile->name); ENDdbgPrintInstInfo @@ -100,10 +97,10 @@ static void prepareFile(instanceData *pData, uchar *newFileName) { if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, pData->fCreateMode); } else { - pData->fd = -1; + pData->fh = -1; /* file does not exist, create it (and eventually parent directories */ if(pData->bCreateDirs) { /* we fist need to create parent dirs if they are missing @@ -119,18 +116,18 @@ static void prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, pData->fCreateMode); - if(pData->fd != -1) { + if(pData->fh != -1) { /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, + if(fchown(pData->fh, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; - close(pData->fd); - pData->fd = -1; + close(pData->fh); + pData->fh = -1; errno = eSave; } /* we will silently ignore the chown() failure @@ -143,20 +140,71 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } #endif -static void -prepareFile(instanceData *pData, uchar *newFileName) +/* ---BEGIN FILE OBJECT---------------------------------------------------- */ +/* This code handles the "file object". This is split from the actual + * instance data, because several instances may write into the same file. + * If so, we need to use a single object, and also synchronize their writes. + * So we keep the file object separately, and just stick a reference into + * the instance data. + */ + +static inline rsRetVal +fileObjConstruct(file_t **ppFile) +{ + file_t *pFile; + DEFiRet; + + CHKmalloc(pFile = malloc(sizeof(file_t))); + pFile->name = NULL; + pFile->hdfsHost = NULL; + pFile->fh = NULL; + pFile->nUsers = 0; + + *ppFile = pFile; +finalize_it: + RETiRet; +} + +static inline void +fileObjAddUser(file_t *pFile) +{ + /* init mutex only when second user is added */ + ++pFile->nUsers; + if(pFile->nUsers == 2) + pthread_mutex_init(&pFile->mut, NULL); +} + +static inline rsRetVal +fileObjDestruct(file_t **ppFile) +{ + file_t *pFile = *ppFile; + if(pFile->nUsers > 1) + pthread_mutex_destroy(&pFile->mut); + fileClose(pFile); + free(pFile->name); + free((char*)pFile->hdfsHost); + free(pFile->fh); + + return RS_RET_OK; +} + +static inline rsRetVal +fileOpen(file_t *pFile) { -# if USE_REGULAR_FS - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0666); -# else - dbgprintf("omhdfs: try to connect to host '%s' at port %d\n", - pData->hdfsHost, pData->hdfsPort); - pData->fs = hdfsConnect(pData->hdfsHost, pData->hdfsPort); - if(pData->fs == NULL) { - dbgprintf("omhdfs: error can not connect to hdfs\n"); + DEFiRet; + + assert(pFile->fh == NULL); + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + DBGPRINTF("omhdfs: try to connect to HDFS at host '%s', port %d\n", + pFile->hdfsHost, pFile->hdfsPort); + pFile->fs = hdfsConnect(pFile->hdfsHost, pFile->hdfsPort); + if(pFile->fs == NULL) { + DBGPRINTF("omhdfs: error can not connect to hdfs\n"); + ABORT_FINALIZE(RS_RET_SUSPENDED); } - pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_APPEND, 0, 0, 0); - if(pData->fd == NULL) { + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_APPEND, 0, 0, 0); + if(pFile->fh == NULL) { /* maybe the file does not exist, so we try to create it now. * Note that we can not use hdfsExists() because of a deficit in * it: https://issues.apache.org/jira/browse/HDFS-1154 @@ -164,68 +212,75 @@ prepareFile(instanceData *pData, uchar *newFileName) * the file does not exist. */ if(errno == ENOENT) { - dbgprintf("omhdfs: ENOENT trying to append to '%s', now trying create\n", - newFileName); - pData->fd = hdfsOpenFile(pData->fs, (char*)newFileName, O_WRONLY|O_CREAT, 0, 0, 0); + DBGPRINTF("omhdfs: ENOENT trying to append to '%s', now trying create\n", + pFile->name); + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_CREAT, 0, 0, 0); } } - if(!pData->fd) { - dbgprintf("omhdfs: failed to open %s for writing!\n", newFileName); - // TODO: suspend/error report + if(pFile->fh == NULL) { + DBGPRINTF("omhdfs: failed to open %s for writing!\n", pFile->name); + ABORT_FINALIZE(RS_RET_SUSPENDED); } -# endif +finalize_it: + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); + RETiRet; } -static rsRetVal writeFile(uchar **ppString, instanceData *pData) + +static inline rsRetVal +fileWrite(file_t *pFile, uchar *buf) { size_t lenWrite; DEFiRet; -# if USE_REGULAR_FS - if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { - int e = errno; - dbgprintf("omhdfs write error!\n"); - - /* If the filesystem is filled up, just ignore - * it for now and continue writing when possible - */ - //if(pData->fileType == eTypeFILE && e == ENOSPC) - //return RS_RET_OK; - - //(void) close(pData->fd); - iRet = RS_RET_DISABLE_ACTION; - errno = e; - //logerror((char*) pData->f_fname); - } -# else - lenWrite = strlen((char*) ppString[0]); - tSize num_written_bytes = hdfsWrite(pData->fs, pData->fd, ppString[0], lenWrite); + assert(pFile->fh != NULL); + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + lenWrite = strlen((char*) buf); + tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { - dbgprintf("omhdfs: failed to write %s, expected %lu bytes, written %lu\n", pData->fileName, - lenWrite, (unsigned long) num_written_bytes); - // TODO: suspend/error report + errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, "omhdfs: failed to write %s, expected %lu bytes, " + "written %lu\n", pFile->name, lenWrite, (unsigned long) num_written_bytes); + ABORT_FINALIZE(RS_RET_SUSPENDED); } -# endif +finalize_it: + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); RETiRet; } +static inline rsRetVal +fileClose(file_t *pFile) +{ + if(pFile->nUsers > 1) + d_pthread_mutex_lock(&pFile->mut); + if(pFile->fh != NULL) { + hdfsCloseFile(pFile->fs, pFile->fh); + pFile->fh = NULL; + } + + if(pFile->nUsers > 1) + d_pthread_mutex_unlock(&pFile->mut); + + return RS_RET_OK; +} + +/* ---END FILE OBJECT---------------------------------------------------- */ + + BEGINcreateInstance CODESTARTcreateInstance - //pData->fd = -1; + pData->pFile = NULL; ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance -# ifdef USE_REGULAR_FS - if(pData->fd != -1) - close(pData->fd); -# else - hdfsCloseFile(pData->fs, pData->fd); -# endif + fileObjDestruct(&pData->pFile); ENDfreeInstance @@ -235,8 +290,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - dbgprintf(" (%s)\n", pData->fileName); - iRet = writeFile(ppString, pData); + DBGPRINTF(" (%s)\n", pData->pFile->name); + iRet = fileWrite(pData->pFile, ppString[0]); ENDdoAction @@ -251,34 +306,29 @@ CODESTARTparseSelectorAct /* ok, if we reach this point, we have something for us */ p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ CHKiRet(createInstance(&pData)); - CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* rgerhards 2004-11-17: from now, we need to have different - * processing, because after the first comma, the template name - * to use is specified. So we need to scan for the first coma first - * and then look at the rest of the line. - */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); - // TODO: check for NULL filename - CHKmalloc(pData->fileName = ustrdup(fileName)); + if(fileName == NULL) { + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: no file name specified, can not continue"); + ABORT_FINALIZE(RS_RET_FILE_NOT_SPECIFIED); + } + + CHKiRet(fileObjConstruct(&pData->pFile)); if(hdfsHost == NULL) { - pData->hdfsHost = "default"; + CHKmalloc(pData->pFile->hdfsHost = strdup("default")); } else { - CHKmalloc(pData->hdfsHost = strdup((char*)hdfsHost)); + CHKmalloc(pData->pFile->hdfsHost = strdup((char*)hdfsHost)); } - pData->hdfsPort = hdfsPort; + pData->pFile->hdfsPort = hdfsPort; - prepareFile(pData, pData->fileName); + fileOpen(pData->pFile); -#if 0 - if ( pData->fd < 0 ){ - pData->fd = -1; - dbgprintf("Error opening log file: %s\n", pData->f_fname); - logerror((char*) pData->f_fname); + if(pData->pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pData->pFile->name); + iRet = RS_RET_SUSPENDED; } -#endif CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -307,6 +357,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINmodExit CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -320,6 +371,8 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); -- cgit From 0ee524c391d017225049542fffe572d7de7d1512 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 09:55:21 +0000 Subject: omhdfs: fixed small bug ... that I could only see on my execution environment (I right now have two envs, a local one where I can compile, but not run and a remote one where I can do both, but this is a bit less convenient to use). --- plugins/omhdfs/omhdfs.c | 1 + 1 file changed, 1 insertion(+) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 71080585..5d28d5dd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -316,6 +316,7 @@ CODESTARTparseSelectorAct } CHKiRet(fileObjConstruct(&pData->pFile)); + CHKmalloc(pData->pFile->name = (uchar*)strdup((char*)fileName)); if(hdfsHost == NULL) { CHKmalloc(pData->pFile->hdfsHost = strdup("default")); } else { -- cgit From 9696cdef34f5d033564138fb9d4afb87daa6b1be Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 12:32:01 +0200 Subject: omhdfs: files now kept inside a hashtable for use by multiple actions Note:compiles, but not yet tested --- plugins/omhdfs/omhdfs.c | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 5d28d5dd..eaca90e4 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -46,6 +46,7 @@ #include "module-template.h" #include "unicode-helper.h" #include "errmsg.h" +#include "hashtable.h" MODULE_TYPE_OUTPUT @@ -54,6 +55,9 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +/* global data */ +static struct hashtable *files; /* holds all file objects that we know */ + /* globals for default values */ static uchar *fileName = NULL; static uchar *hdfsHost = NULL; @@ -172,6 +176,7 @@ fileObjAddUser(file_t *pFile) ++pFile->nUsers; if(pFile->nUsers == 2) pthread_mutex_init(&pFile->mut, NULL); + dbgprintf("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); } static inline rsRetVal @@ -296,6 +301,9 @@ ENDdoAction BEGINparseSelectorAct + file_t *pFile; + int r; + uchar *keybuf; CODESTARTparseSelectorAct /* first check if this config line is actually for us */ @@ -315,21 +323,28 @@ CODESTARTparseSelectorAct ABORT_FINALIZE(RS_RET_FILE_NOT_SPECIFIED); } - CHKiRet(fileObjConstruct(&pData->pFile)); - CHKmalloc(pData->pFile->name = (uchar*)strdup((char*)fileName)); - if(hdfsHost == NULL) { - CHKmalloc(pData->pFile->hdfsHost = strdup("default")); - } else { - CHKmalloc(pData->pFile->hdfsHost = strdup((char*)hdfsHost)); + pFile = hashtable_search(files, fileName); + if(pFile == NULL) { + /* we need a new file object, this one not seen before */ + CHKiRet(fileObjConstruct(&pFile)); + CHKmalloc(pFile->name = (uchar*)strdup((char*)fileName)); + CHKmalloc(keybuf = ustrdup(fileName)); + r = hashtable_insert(files, keybuf, pFile); + if(r == 0) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } - pData->pFile->hdfsPort = hdfsPort; + fileObjAddUser(pFile); - fileOpen(pData->pFile); - - if(pData->pFile->fh == NULL){ - errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pData->pFile->name); + CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); + pFile->hdfsPort = hdfsPort; + fileOpen(pFile); + if(pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); iRet = RS_RET_SUSPENDED; } + + pData->pFile = pFile; + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -359,6 +374,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINmodExit CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); + if(files != NULL) + hashtable_destroy(files, 1); /* 1 => free all values automatically */ ENDmodExit @@ -373,6 +390,7 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); -- cgit From 255895a58b3f2a54fecf971da700caf265b4e1f0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 13:59:48 +0000 Subject: omhdfs: more improvements finally this looks almost production ready for files where no directory path needs to be created --- plugins/imuxsock/imuxsock.c | 4 ++-- plugins/omhdfs/Makefile.am | 3 +-- plugins/omhdfs/omhdfs.c | 47 +++++++++++++++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 18 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 24bcebb7..ad2d61c8 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -278,7 +278,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); if(ratelimitInterval > 0) { - if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); @@ -755,7 +755,7 @@ CODESTARTwillRun if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; if(ratelimitIntervalSysSock > 0) { - if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn)) == NULL) { + if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); diff --git a/plugins/omhdfs/Makefile.am b/plugins/omhdfs/Makefile.am index 2e7ef8ea..95e6b102 100644 --- a/plugins/omhdfs/Makefile.am +++ b/plugins/omhdfs/Makefile.am @@ -1,7 +1,6 @@ pkglib_LTLIBRARIES = omhdfs.la omhdfs_la_SOURCES = omhdfs.c -#omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -I/usr/lib/jvm/java-6-sun/include -I/usr/lib/jvm/java-6-sun/include/linux omhdfs_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(JAVA_INCLUDES) omhdfs_la_LDFLAGS = -module -avoid-version -lhdfs $(JAVA_LIBS) -omhdfs_la_LIBADD = +omhdfs_la_LIBADD = $(RSRT_LIBS) diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index eaca90e4..734c28cd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -61,6 +61,7 @@ static struct hashtable *files; /* holds all file objects that we know */ /* globals for default values */ static uchar *fileName = NULL; static uchar *hdfsHost = NULL; +static uchar *dfltTplName = NULL; /* default template name to use */ int hdfsPort = 0; /* end globals for default values */ @@ -176,10 +177,10 @@ fileObjAddUser(file_t *pFile) ++pFile->nUsers; if(pFile->nUsers == 2) pthread_mutex_init(&pFile->mut, NULL); - dbgprintf("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); + DBGPRINTF("omhdfs: file %s now being used by %d actions\n", pFile->name, pFile->nUsers); } -static inline rsRetVal +static rsRetVal fileObjDestruct(file_t **ppFile) { file_t *pFile = *ppFile; @@ -193,6 +194,18 @@ fileObjDestruct(file_t **ppFile) return RS_RET_OK; } +/* this function is to be used as destructor for the + * hash table code. + */ +static void +fileObjDestruct4Hashtable(void *ptr) +{ + dbgprintf("omfile: fileObjDestruct4Hashtable called\n"); + file_t *pFile = (file_t*) ptr; + fileObjDestruct(&pFile); +} + + static inline rsRetVal fileOpen(file_t *pFile) { @@ -261,6 +274,8 @@ finalize_it: static inline rsRetVal fileClose(file_t *pFile) { + DEFiRet; + if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); if(pFile->fh != NULL) { @@ -271,7 +286,7 @@ fileClose(file_t *pFile) if(pFile->nUsers > 1) d_pthread_mutex_unlock(&pFile->mut); - return RS_RET_OK; + RETiRet; } /* ---END FILE OBJECT---------------------------------------------------- */ @@ -285,7 +300,8 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - fileObjDestruct(&pData->pFile); + if(pData->pFile != NULL) + fileObjDestruct(&pData->pFile); ENDfreeInstance @@ -315,8 +331,8 @@ CODESTARTparseSelectorAct p += sizeof(":omhdfs:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ CHKiRet(createInstance(&pData)); CODE_STD_STRING_REQUESTparseSelectorAct(1) - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); - //(pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName)); + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, + (dfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : dfltTplName)); if(fileName == NULL) { errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: no file name specified, can not continue"); @@ -327,17 +343,18 @@ CODESTARTparseSelectorAct if(pFile == NULL) { /* we need a new file object, this one not seen before */ CHKiRet(fileObjConstruct(&pFile)); - CHKmalloc(pFile->name = (uchar*)strdup((char*)fileName)); + CHKmalloc(pFile->name = fileName); CHKmalloc(keybuf = ustrdup(fileName)); + fileName = NULL; /* re-set, data passed to file object */ + CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); + pFile->hdfsPort = hdfsPort; + fileOpen(pFile); r = hashtable_insert(files, keybuf, pFile); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } fileObjAddUser(pFile); - CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); - pFile->hdfsPort = hdfsPort; - fileOpen(pFile); if(pFile->fh == NULL){ errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); iRet = RS_RET_SUSPENDED; @@ -390,11 +407,13 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); - CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string)); + CHKmalloc(files = create_hashtable(20, hash_from_string, key_equals_string, + fileObjDestruct4Hashtable)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsfilename", 0, eCmdHdlrGetWord, NULL, &fileName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfshost", 0, eCmdHdlrGetWord, NULL, &hdfsHost, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsdefaulttemplate", 0, eCmdHdlrGetWord, NULL, &dfltTplName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr ENDmodInit -- cgit From 670e81c9a8275a5509efd71dae66d9a267ec1574 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 14:36:39 +0000 Subject: omhdfs: made action suspend/resume working --- plugins/omhdfs/omhdfs.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 734c28cd..4fbf2ef4 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -276,16 +276,19 @@ fileClose(file_t *pFile) { DEFiRet; + if(pFile->fh == NULL) + FINALIZE; + if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); - if(pFile->fh != NULL) { - hdfsCloseFile(pFile->fs, pFile->fh); - pFile->fh = NULL; - } + + hdfsCloseFile(pFile->fs, pFile->fh); + pFile->fh = NULL; if(pFile->nUsers > 1) d_pthread_mutex_unlock(&pFile->mut); +finalize_it: RETiRet; } @@ -307,12 +310,26 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + fileClose(pData->pFile); + fileOpen(pData->pFile); + if(pData->pFile->fh == NULL){ + dbgprintf("omhdfs: tried to resume file %s, but still no luck...\n", + pData->pFile->name); + iRet = RS_RET_SUSPENDED; + } ENDtryResume BEGINdoAction CODESTARTdoAction DBGPRINTF(" (%s)\n", pData->pFile->name); + if(pData->pFile->fh == NULL) { + fileOpen(pData->pFile); + if(pData->pFile->fh == NULL) { + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } iRet = fileWrite(pData->pFile, ppString[0]); +finalize_it: ENDdoAction @@ -349,17 +366,16 @@ CODESTARTparseSelectorAct CHKmalloc(pFile->hdfsHost = strdup((hdfsHost == NULL) ? "default" : (char*) hdfsHost)); pFile->hdfsPort = hdfsPort; fileOpen(pFile); + if(pFile->fh == NULL){ + errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - " + "retrying later", pFile->name); + iRet = RS_RET_SUSPENDED; + } r = hashtable_insert(files, keybuf, pFile); if(r == 0) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } fileObjAddUser(pFile); - - if(pFile->fh == NULL){ - errmsg.LogError(0, RS_RET_ERR_HDFS_OPEN, "omhdfs: failed to open %s - retrying later", pFile->name); - iRet = RS_RET_SUSPENDED; - } - pData->pFile = pFile; CODE_STD_FINALIZERparseSelectorAct -- cgit From d5f16404f93d54afddebb9fb683469fc712d2335 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 17:26:14 +0200 Subject: omhdfs: added doc --- plugins/omhdfs/omhdfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 4fbf2ef4..42ed834f 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -1,5 +1,5 @@ /* omhdfs.c - * This is the implementation of the build-in file output module. + * This is an output module to support Hadoop's HDFS. * * NOTE: read comments in module-template.h to understand how this file * works! -- cgit From 2cc5fbd49fdca20cdeb8eb81455bc1baef627a77 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 09:43:34 +0200 Subject: bugfix/imuxsock: system log socket was deleted in systemd-mode, too related bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=200 Note that this may not be the ultimate fix. I could not definitely identify the root cause of the problem and so I did some changes that I *think* (but could not verify) that resolved the issue. --- plugins/imuxsock/imuxsock.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 24bcebb7..906e28e6 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -139,6 +139,7 @@ static int startIndexUxLocalSockets; /* process fd from that index on (used to * read-only after startup */ static int nfd = 1; /* number of Unix sockets open / read-only after startup */ +static int bSysSockFromSystemd = 0; /* Did we receive the system socket from systemd? */ /* config settings */ static int bOmitLocalLogging = 0; @@ -368,6 +369,7 @@ openLogSocket(lstn_t *pLstn) return -1; if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { + bSysSockFromSystemd = 1; /* set default */ int r; /* System log socket code. Check whether an FD was passed in from systemd. If @@ -396,6 +398,7 @@ openLogSocket(lstn_t *pLstn) errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } + bSysSockFromSystemd = 1; /* indicate we got the socket from systemd */ } else { CHKiRet(createLogSocket(pLstn)); } @@ -415,15 +418,6 @@ openLogSocket(lstn_t *pLstn) } } -#if 0 - // SO_TIMESTAMP currently does not work for an unknown reason. Any help is appreciated! - one = 1; - if(setsockopt(pLstn->fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { - errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP '%s'", pLstn->sockName); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } -#endif - finalize_it: if(iRet != RS_RET_OK) { close(pLstn->fd); @@ -802,13 +796,16 @@ CODESTARTafterRun /* Clean-up files. If systemd passed us a socket it is * systemd's job to clean it up.*/ - if (sd_listen_fds(0) > 0) + if(bSysSockFromSystemd) { + DBGPRINTF("imuxsock: got system socket from systemd, not unlinking it\n"); i = 1; - else + } else i = startIndexUxLocalSockets; for(; i < nfd; i++) - if (listeners[i].sockName && listeners[i].fd != -1) + if (listeners[i].sockName && listeners[i].fd != -1) { + DBGPRINTF("imuxsock: unlinking unix socket file[%d] %s\n", i, listeners[i].sockName); unlink((char*) listeners[i].sockName); + } /* free no longer needed string */ free(pLogSockName); free(pLogHostName); -- cgit From e40cb595a2da943483124fae8f215a397add9fca Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 13:58:41 +0200 Subject: omhdfs: support for HUP added --- plugins/omhdfs/omhdfs.c | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 42ed834f..eefea722 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -47,6 +47,7 @@ #include "unicode-helper.h" #include "errmsg.h" #include "hashtable.h" +#include "hashtable_itr.h" MODULE_TYPE_OUTPUT @@ -200,7 +201,6 @@ fileObjDestruct(file_t **ppFile) static void fileObjDestruct4Hashtable(void *ptr) { - dbgprintf("omfile: fileObjDestruct4Hashtable called\n"); file_t *pFile = (file_t*) ptr; fileObjDestruct(&pFile); } @@ -214,6 +214,7 @@ fileOpen(file_t *pFile) assert(pFile->fh == NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); + DBGPRINTF("omhdfs: try to connect to HDFS at host '%s', port %d\n", pFile->hdfsHost, pFile->hdfsPort); pFile->fs = hdfsConnect(pFile->hdfsHost, pFile->hdfsPort); @@ -256,6 +257,17 @@ fileWrite(file_t *pFile, uchar *buf) assert(pFile->fh != NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); + + /* open file if not open. This must be done *here* and while mutex-protected + * because of HUP handling (which is async to normal processing!). + */ + if(pFile->fh == NULL) { + fileOpen(pFile); + if(pFile->fh == NULL) { + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } + lenWrite = strlen((char*) buf); tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { @@ -321,15 +333,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->pFile->name); - if(pData->pFile->fh == NULL) { - fileOpen(pData->pFile); - if(pData->pFile->fh == NULL) { - ABORT_FINALIZE(RS_RET_SUSPENDED); - } - } + DBGPRINTF("omuxsock: action to to write to %s\n", pData->pFile->name); iRet = fileWrite(pData->pFile, ppString[0]); -finalize_it: ENDdoAction @@ -382,6 +387,24 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct +BEGINdoHUP + file_t *pFile; + struct hashtable_itr *itr; +CODESTARTdoHUP + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(files); + if(hashtable_count(files) > 0) + { + do { + pFile = (file_t *) hashtable_iterator_value(itr); + fileClose(pFile); + DBGPRINTF("imuxsock: HUP, closing file %s\n", pFile->name); + } while (hashtable_iterator_advance(itr)); + } +ENDdoHUP + + /* Reset config variables for this module to default values. * rgerhards, 2007-07-17 */ @@ -415,6 +438,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_doHUP ENDqueryEtryPt -- cgit From 4b9a92bc725f7436b7958e673a9665a90b548e86 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 12:46:39 +0000 Subject: omhdfs: fixed some issues with previous commit --- plugins/omhdfs/omhdfs.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index eefea722..b075432d 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -254,7 +254,6 @@ fileWrite(file_t *pFile, uchar *buf) size_t lenWrite; DEFiRet; - assert(pFile->fh != NULL); if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); @@ -391,17 +390,18 @@ BEGINdoHUP file_t *pFile; struct hashtable_itr *itr; CODESTARTdoHUP - /* Iterator constructor only returns a valid iterator if - * the hashtable is not empty */ - itr = hashtable_iterator(files); - if(hashtable_count(files) > 0) - { - do { - pFile = (file_t *) hashtable_iterator_value(itr); - fileClose(pFile); - DBGPRINTF("imuxsock: HUP, closing file %s\n", pFile->name); - } while (hashtable_iterator_advance(itr)); - } + DBGPRINTF("omhdfs: HUP received (file count %d)\n", hashtable_count(files)); + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(files); + if(hashtable_count(files) > 0) + { + do { + pFile = (file_t *) hashtable_iterator_value(itr); + fileClose(pFile); + DBGPRINTF("omhdfs: HUP, closing file %s\n", pFile->name); + } while (hashtable_iterator_advance(itr)); + } ENDdoHUP -- cgit From bbf38b082a17c77dfa86dcdbd0b7b7b6d36c07b6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 16:17:49 +0200 Subject: imuxsock: fixed dumb mistake of last commit I left a testing setting in, resulting in /dev/log never being deleted... --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 906e28e6..75a6b00b 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -369,7 +369,7 @@ openLogSocket(lstn_t *pLstn) return -1; if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { - bSysSockFromSystemd = 1; /* set default */ + bSysSockFromSystemd = 0; /* set default */ int r; /* System log socket code. Check whether an FD was passed in from systemd. If -- cgit From a627ac99ba2c3404ca926a19fb06cbd6f43b53c8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 16:25:49 +0200 Subject: omhdfs: added ability to create non-existing directories in name path --- plugins/omhdfs/omhdfs.c | 128 +++++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 56 deletions(-) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index b075432d..9705b7fd 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -97,54 +97,41 @@ CODESTARTdbgPrintInstInfo ENDdbgPrintInstInfo - -#if 0 -static void prepareFile(instanceData *pData, uchar *newFileName) +/* note that hdfsFileExists() does not work, so we did our + * own function to see if a pathname exists. Returns 0 if the + * file does not exists, something else otherwise. Note that + * we can also check a directroy (if that matters...) + */ +static int +HDFSFileExists(hdfsFS fs, uchar *name) { - if(access((char*)newFileName, F_OK) == 0) { - /* file already exists */ - pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, - pData->fCreateMode); + int r; + hdfsFileInfo *info; + + info = hdfsGetPathInfo(fs, (char*)name); + /* if things go wrong, we assume it is because the file + * does not exist. We do not get too much information... + */ + if(info == NULL) { + r = 0; } else { - pData->fh = -1; - /* file does not exist, create it (and eventually parent directories */ - if(pData->bCreateDirs) { - /* we fist need to create parent dirs if they are missing - * We do not report any errors here ourselfs but let the code - * fall through to error handler below. - */ - if(makeFileParentDirs(newFileName, strlen((char*)newFileName), - pData->fDirCreateMode, pData->dirUID, - pData->dirGID, pData->bFailOnChown) != 0) { - return; /* we give up */ - } - } - /* no matter if we needed to create directories or not, we now try to create - * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) - */ - pData->fh = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, - pData->fCreateMode); - if(pData->fh != -1) { - /* check and set uid/gid */ - if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { - /* we need to set owner/group */ - if(fchown(pData->fh, pData->fileUID, - pData->fileGID) != 0) { - if(pData->bFailOnChown) { - int eSave = errno; - close(pData->fh); - pData->fh = -1; - errno = eSave; - } - /* we will silently ignore the chown() failure - * if configured to do so. - */ - } - } - } + r = 1; + hdfsFreeFileInfo(info, 1); } + return r; +} + +static inline rsRetVal +HDFSmkdir(hdfsFS fs, uchar *name) +{ + DEFiRet; + if(hdfsCreateDirectory(fs, (char*)name) == -1) + ABORT_FINALIZE(RS_RET_ERR); + +finalize_it: + RETiRet; } -#endif + /* ---BEGIN FILE OBJECT---------------------------------------------------- */ /* This code handles the "file object". This is split from the actual @@ -195,6 +182,42 @@ fileObjDestruct(file_t **ppFile) return RS_RET_OK; } + +/* check, and potentially create, all names inside a path */ +static rsRetVal +filePrepare(file_t *pFile) +{ + uchar *p; + uchar *pszWork; + size_t len; + DEFiRet; + + if(HDFSFileExists(pFile->fs, pFile->name)) + FINALIZE; + + /* file does not exist, create it (and eventually parent directories */ + if(1) { // check if bCreateDirs + len = ustrlen(pFile->name) + 1; + CHKmalloc(pszWork = MALLOC(sizeof(uchar) * len)); + memcpy(pszWork, pFile->name, len); + for(p = pszWork+1 ; *p ; p++) + if(*p == '/') { + /* temporarily terminate string, create dir and go on */ + *p = '\0'; + if(!HDFSFileExists(pFile->fs, pszWork)) { + CHKiRet(HDFSmkdir(pFile->fs, pszWork)); + } + *p = '/'; + } + free(pszWork); + return 0; + } + +finalize_it: + RETiRet; +} + + /* this function is to be used as destructor for the * hash table code. */ @@ -222,6 +245,9 @@ fileOpen(file_t *pFile) DBGPRINTF("omhdfs: error can not connect to hdfs\n"); ABORT_FINALIZE(RS_RET_SUSPENDED); } + + CHKiRet(filePrepare(pFile)); + pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_APPEND, 0, 0, 0); if(pFile->fh == NULL) { /* maybe the file does not exist, so we try to create it now. @@ -271,7 +297,8 @@ fileWrite(file_t *pFile, uchar *buf) tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); if((unsigned) num_written_bytes != lenWrite) { errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, "omhdfs: failed to write %s, expected %lu bytes, " - "written %lu\n", pFile->name, lenWrite, (unsigned long) num_written_bytes); + "written %lu\n", pFile->name, (unsigned long) lenWrite, + (unsigned long) num_written_bytes); ABORT_FINALIZE(RS_RET_SUSPENDED); } @@ -410,17 +437,6 @@ ENDdoHUP */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { -/* - fileUID = -1; - fileGID = -1; - dirUID = -1; - dirGID = -1; - bFailOnChown = 1; - iDynaFileCacheSize = 10; - fCreateMode = 0644; - fDirCreateMode = 0700; - bCreateDirs = 1; -*/ hdfsHost = NULL; hdfsPort = 0; return RS_RET_OK; -- cgit From c0de8c6dc4ec4ac270b0c313ea5a1d2689b44db3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 6 Oct 2010 12:37:24 +0200 Subject: bugfix/imuxsock: did not compile on platforms without SCM_CREDENTIALS Note: we do rate-limiting by the pid and we obtain the pid via SCM_CREDENTIALS socket options. So this patch effectively disables the (new) ratelimiting capability. In a later patch, I'll see that I can provide a global ratelimiting capability, which could always be used (an alternative may be using the tag, will check this out as well). --- plugins/imuxsock/imuxsock.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 41bff4f9..5b548602 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -406,6 +406,7 @@ openLogSocket(lstn_t *pLstn) CHKiRet(createLogSocket(pLstn)); } +# if HAVE_SCM_CREDENTIALS if(pLstn->bUseCreds) { one = 1; if(setsockopt(pLstn->fd, SOL_SOCKET, SO_PASSCRED, &one, (socklen_t) sizeof(one)) != 0) { @@ -417,6 +418,9 @@ openLogSocket(lstn_t *pLstn) pLstn->bUseCreds = 0; } } +# else /* HAVE_SCM_CREDENTIALS */ + pLstn->bUseCreds = 0; +# endif /* HAVE_SCM_CREDENTIALS */ finalize_it: if(iRet != RS_RET_OK) { @@ -513,7 +517,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) rs_ratelimit_state_t *ratelimiter = NULL; DEFiRet; -// TODO: handle format errors?? + /* TODO: handle format errors?? */ /* we need to parse the pri first, because we need the severity for * rate-limiting as well. */ @@ -530,8 +534,10 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) facil = LOG_FAC(pri); sever = LOG_PRI(pri); - if(sever >= pLstn->ratelimitSev) + if(sever >= pLstn->ratelimitSev) { + /* note: if cred == NULL, then ratelimiter == NULL as well! */ findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ + } datetime.getCurrTime(&st, &tt); if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt, cred->pid)) { @@ -637,6 +643,7 @@ static rsRetVal readSocket(lstn_t *pLstn) dbgprintf("Message from UNIX socket: #%d\n", pLstn->fd); if(iRcvd > 0) { cred = NULL; +# if HAVE_SCM_CREDENTIALS if(pLstn->bUseCreds) { dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { @@ -644,11 +651,12 @@ static rsRetVal readSocket(lstn_t *pLstn) if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { cred = (struct ucred*) CMSG_DATA(cm); dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); - //break; + break; } } dbgprintf("XXX: post CM loop\n"); } +# endif /* HAVE_SCM_CREDENTIALS */ CHKiRet(SubmitMsg(pRcv, iRcvd, pLstn, cred)); } else if(iRcvd < 0 && errno != EINTR) { char errStr[1024]; -- cgit From d51d6847a84eeec573ce2dd4feeee991d3eb0bcc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 6 Oct 2010 12:54:33 +0200 Subject: previous bugfix was incomplete, resolving this --- plugins/imuxsock/imuxsock.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 5b548602..566dde1b 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -66,6 +66,10 @@ MODULE_TYPE_INPUT #endif #endif +/* emulate struct ucred for platforms that do not have it */ +#ifndef HAVE_SCM_CREDENTIALS +struct ucred { int pid; }; +#endif /* handle some defines missing on more than one platform */ #ifndef SUN_LEN -- cgit From 8c3d40b798e3ed68fb03629d87b55140cb6bc044 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 Oct 2010 17:11:00 +0200 Subject: removed need for math library by doing math a little bit more optimal in hash table code. Also reduced memory requirement for imuxsock hash tables (expected number of connections was set too high -- table can be extended dynamically). --- plugins/imuxsock/imuxsock.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 566dde1b..0eee1122 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -283,7 +283,7 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) } CHKiRet(prop.ConstructFinalize(listeners[nfd].hostName)); if(ratelimitInterval > 0) { - if((listeners[nfd].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { + if((listeners[nfd].ht = create_hashtable(100, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); @@ -761,7 +761,7 @@ CODESTARTwillRun if(pLogSockName != NULL) listeners[0].sockName = pLogSockName; if(ratelimitIntervalSysSock > 0) { - if((listeners[0].ht = create_hashtable(1000, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { + if((listeners[0].ht = create_hashtable(100, hash_from_key_fn, key_equals_fn, NULL)) == NULL) { /* in this case, we simply turn of rate-limiting */ dbgprintf("imuxsock: turning off rate limiting because we could not " "create hash table\n"); -- cgit From ed9fd3541f07929008a64f5b6ea3bcce70e62c25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 15 Oct 2010 08:23:29 +0200 Subject: imfile: bug fixes either one or two bugs fixed ;) Definitely a problem where no state file is written when working with relative pathes. Also, some problems with offsets should be fixed for very large files. However, I could not yet experimentally show the issue so it probably needs more verification. --- plugins/imfile/imfile.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 8a10e26f..e067014e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -337,12 +337,15 @@ persistStrmState(fileInfo_t *pInfo) { DEFiRet; strm_t *psSF = NULL; /* state file (stream) */ + size_t lenDir; ASSERT(pInfo != NULL); /* TODO: create a function persistObj in obj.c? */ CHKiRet(strm.Construct(&psSF)); - CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + lenDir = ustrlen(glbl.GetWorkDir()); + if(lenDir > 0) + CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), lenDir)); CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); -- cgit From ffd08f2a6caaaddb86ccbec4206bf560d34fcfd7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 24 Oct 2010 14:31:12 +0200 Subject: imfile improvements - added the $InputFilePersistStateInterval config directive to imfile - changed imfile so that the state file is never deleted (makes imfile more robust in regard to fatal failures) --- plugins/imfile/imfile.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) (limited to 'plugins') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 3981f9f7..91493bb1 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -67,15 +67,21 @@ typedef struct fileInfo_s { uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; + int nRecords; /**< How many records did we process before persisting the stream? */ + int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ } fileInfo_t; +/* forward definitions */ +static rsRetVal persistStrmState(fileInfo_t *pInfo); + /* config variables */ static uchar *pszFileName = NULL; static uchar *pszFileTag = NULL; static uchar *pszStateFile = NULL; static int iPollInterval = 10; /* number of seconds to sleep when there was no file activity */ +static int iPersistStateInterval = 0; /* how often if state file to be persisted? (default 0->never) */ static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ @@ -154,10 +160,9 @@ openFile(fileInfo_t *pThis) CHKiRet(strm.SeekCurrOffs(pThis->pStrm)); - /* OK, we could successfully read the file, so we now can request that it be deleted. - * If we need it again, it will be written on the next shutdown. + /* note: we do not delete the state file, so that the last position remains + * known even in the case that rsyslogd aborts for some reason (like powerfail) */ - psSF->bDeleteOnClose = 1; finalize_it: if(psSF != NULL) @@ -211,6 +216,10 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ + if(pThis->iPersistStateInterval > 0 && pThis->nRecords++ >= pThis->iPersistStateInterval) { + persistStrmState(pThis); + pThis->nRecords = 0; + } } finalize_it: @@ -244,27 +253,12 @@ finalize_it: * IMPORTANT: the calling interface of this function can NOT be modified. It actually is * called by pthreads. The provided argument is currently not being used. */ -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ static void inputModuleCleanup(void __attribute__((unused)) *arg) { BEGINfunc -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ - - - - /* so far not needed */ - - - -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ ENDfunc } -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ /* This function is called by the framework to gather the input. The module stays @@ -499,6 +493,9 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->iSeverity = iSeverity; pThis->iFacility = iFacility; + pThis->iPersistStateInterval = iPersistStateInterval; + pThis->nRecords = 0; + iPersistStateInterval = 0; } else { errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one"); ABORT_FINALIZE(RS_RET_OUT_OF_DESRIPTORS); @@ -544,6 +541,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepollinterval", 0, eCmdHdlrInt, NULL, &iPollInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, + NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); /* that command ads a new file! */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrunfilemonitor", 0, eCmdHdlrGetWord, addMonitor, NULL, STD_LOADABLE_MODULE_ID)); -- cgit From 0b18c17b2850152203ce9db648ce06212ab67157 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Tue, 30 Nov 2010 13:04:58 +0100 Subject: Fix a potential missing '\0' on too long strings. By implementing a trivial strlcpy it's much easier to detect string truncations and react to them. This also gives a noticeable speedup in buffer handling (can be HUGE), since strlcpy() doesn't clear all the buffer entry before writing data. Converted all uses of strncpy() into strlcpy(). Also, we don't need to check for some null pointers, as there are no malloc-like operations in the doAction loop. --- plugins/omoracle/omoracle.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'plugins') diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 48ee1fa4..30b5834b 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -127,6 +127,13 @@ typedef struct _instanceData { struct oracle_batch batch; } instanceData; +/* To be honest, strlcpy is faster than strncpy and makes very easy to + * detect if a message has been truncated. */ +#ifndef strlcpy +#define strlcpy(dst,src,sz) snprintf((dst), (sz), "%s", (src)) +#endif + + /** Database name, to be filled by the $OmoracleDB directive */ static char* db_name; /** Database user name, to be filled by the $OmoracleDBUser @@ -529,7 +536,7 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction - int i; + int i, sz; char **params = (char**) ppString[0]; CODESTARTdoAction @@ -540,9 +547,13 @@ CODESTARTdoAction for (i = 0; i < pData->batch.arguments && params[i]; i++) { dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); - strncpy(pData->batch.parameters[i][pData->batch.n], params[i], - pData->batch.param_size); - CHKmalloc(pData->batch.parameters[i][pData->batch.n]); + sz = strlcpy(pData->batch.parameters[i][pData->batch.n], + params[i], pData->batch.param_size); + if (sz >= pData->batch.param_size) + errmsg.LogError(0, NO_ERRCODE, + "Possibly truncated %d column of '%s' " + "statement: %s", i, + pData->txt_statement, params[i]); } pData->batch.n++; -- cgit From cc8237736d11b54a3d6089d836da7ccb6972a29c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Dec 2010 12:21:17 +0100 Subject: added $IMUDPSchedulingPolicy and $IMUDPSchedulingPriority config settings --- plugins/imudp/imudp.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'plugins') diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index d76f3544..fea789ec 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -72,6 +72,8 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * termination if we can not get it. -- rgerhards, 2007-12-27 */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ +static uchar *pszSchedPolicy = NULL; /**< scheduling policy (string) */ +static int iSchedPrio = -1; /**< scheduling priority (must not be negative) */ // TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -357,6 +359,7 @@ ENDrunInput /* initialize and return if will run or not */ BEGINwillRun + struct sched_param sparam; CODESTARTwillRun /* we need to create the inputName property (only once during our lifetime) */ CHKiRet(prop.Construct(&pInputName)); @@ -364,6 +367,41 @@ CODESTARTwillRun CHKiRet(prop.ConstructFinalize(pInputName)); net.PrintAllowedSenders(1); /* UDP */ + + if(pszSchedPolicy == NULL) { + if(iSchedPrio != -1) { + errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy not set, but " + "priority - ignoring settings"); + } + } else { + if(iSchedPrio == -1) { + errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy set, but no " + "priority - ignoring settings"); + } + sparam.sched_priority = iSchedPrio; + dbgprintf("imudp trying to set sched policy to '%s', prio %d\n", + pszSchedPolicy, iSchedPrio); + if(0) { /* trick to use conditional compilation */ +# ifdef SCHED_FIFO + } else if(!strcasecmp((char*)pszSchedPolicy, "fifo")) { + pthread_setschedparam(pthread_self(), SCHED_FIFO, &sparam); +# endif +# ifdef SCHED_RR + } else if(!strcasecmp((char*)pszSchedPolicy, "rr")) { + pthread_setschedparam(pthread_self(), SCHED_RR, &sparam); +# endif +# ifdef SCHED_OTHER + } else if(!strcasecmp((char*)pszSchedPolicy, "other")) { + pthread_setschedparam(pthread_self(), SCHED_OTHER, &sparam); +# endif + } else { + errmsg.LogError(errno, NO_ERRCODE, "imudp: invliad scheduling policy '%s' " + "ignoring settings", pszSchedPolicy); + } + free(pszSchedPolicy); + pszSchedPolicy = NULL; + } + /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) @@ -445,6 +483,10 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpolicy", 0, eCmdHdlrGetWord, + NULL, &pszSchedPolicy, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpriority", 0, eCmdHdlrInt, + NULL, &iSchedPrio, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit From 8d862730d297cbf88d9301b7a4cbee47a3f8fb9e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Dec 2010 14:59:44 +0100 Subject: added support for systems without epoll_create1() --- plugins/imptcp/imptcp.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 93906ba0..2afb340f 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -30,6 +30,13 @@ * A copy of the GPL can be found in the file "COPYING" in this distribution. */ #include "config.h" +#if !defined(HAVE_EPOLL_CREATE) +# error imptcp requires OS support for epoll - can not build + /* imptcp gains speed by using modern Linux capabilities. As such, + * it can only be build on platforms supporting the epoll API. + */ +#endif + #include #include #include @@ -1040,7 +1047,18 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } - if((epollfd = epoll_create1(EPOLL_CLOEXEC)) < 0) { +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imptcp uses epoll_create1()\n"); + epollfd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("imptcp uses epoll_create()\n"); + /* reading the docs, the number of epoll events passed to + * epoll_create() seems not to be used at all in kernels. So + * we just provide "a" number, happens to be 10. + */ + epollfd = epoll_create(10); +# endif + if(epollfd < 0) { errmsg.LogError(0, RS_RET_EPOLL_CR_FAILED, "error: epoll_create() failed"); ABORT_FINALIZE(RS_RET_NO_RUN); } -- cgit From dec726ed46a75750b30117ba772fc82e8660db1d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 10 Jan 2011 12:09:38 +0100 Subject: imudp: removing new scheduling priority setting code as it had too many implications. This will now be part of v5 only. --- plugins/imudp/imudp.c | 42 ------------------------------------------ 1 file changed, 42 deletions(-) (limited to 'plugins') diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index fea789ec..d76f3544 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -72,8 +72,6 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * termination if we can not get it. -- rgerhards, 2007-12-27 */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -static uchar *pszSchedPolicy = NULL; /**< scheduling policy (string) */ -static int iSchedPrio = -1; /**< scheduling priority (must not be negative) */ // TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -359,7 +357,6 @@ ENDrunInput /* initialize and return if will run or not */ BEGINwillRun - struct sched_param sparam; CODESTARTwillRun /* we need to create the inputName property (only once during our lifetime) */ CHKiRet(prop.Construct(&pInputName)); @@ -367,41 +364,6 @@ CODESTARTwillRun CHKiRet(prop.ConstructFinalize(pInputName)); net.PrintAllowedSenders(1); /* UDP */ - - if(pszSchedPolicy == NULL) { - if(iSchedPrio != -1) { - errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy not set, but " - "priority - ignoring settings"); - } - } else { - if(iSchedPrio == -1) { - errmsg.LogError(errno, NO_ERRCODE, "imudp: scheduling policy set, but no " - "priority - ignoring settings"); - } - sparam.sched_priority = iSchedPrio; - dbgprintf("imudp trying to set sched policy to '%s', prio %d\n", - pszSchedPolicy, iSchedPrio); - if(0) { /* trick to use conditional compilation */ -# ifdef SCHED_FIFO - } else if(!strcasecmp((char*)pszSchedPolicy, "fifo")) { - pthread_setschedparam(pthread_self(), SCHED_FIFO, &sparam); -# endif -# ifdef SCHED_RR - } else if(!strcasecmp((char*)pszSchedPolicy, "rr")) { - pthread_setschedparam(pthread_self(), SCHED_RR, &sparam); -# endif -# ifdef SCHED_OTHER - } else if(!strcasecmp((char*)pszSchedPolicy, "other")) { - pthread_setschedparam(pthread_self(), SCHED_OTHER, &sparam); -# endif - } else { - errmsg.LogError(errno, NO_ERRCODE, "imudp: invliad scheduling policy '%s' " - "ignoring settings", pszSchedPolicy); - } - free(pszSchedPolicy); - pszSchedPolicy = NULL; - } - /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) @@ -483,10 +445,6 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpolicy", 0, eCmdHdlrGetWord, - NULL, &pszSchedPolicy, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpriority", 0, eCmdHdlrInt, - NULL, &iSchedPrio, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit From 7742b2182fc017ca4c9fcfe3b26f4c8d68a9bd58 Mon Sep 17 00:00:00 2001 From: Dražen Kačar Date: Mon, 10 Jan 2011 12:39:23 +0100 Subject: improved imudp real-time scheduling support & bugfix The original code had quite some issues, which are fixed by this commit. Also we do more error checking now. Signed-off-by: Rainer Gerhards --- plugins/imudp/Makefile.am | 2 +- plugins/imudp/imudp.c | 135 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imudp/Makefile.am b/plugins/imudp/Makefile.am index 517b1287..bc64b8c8 100644 --- a/plugins/imudp/Makefile.am +++ b/plugins/imudp/Makefile.am @@ -3,4 +3,4 @@ pkglib_LTLIBRARIES = imudp.la imudp_la_SOURCES = imudp.c imudp_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) imudp_la_LDFLAGS = -module -avoid-version -imudp_la_LIBADD = +imudp_la_LIBADD = $(IMUDP_LIBS) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 99b69731..ad39ead0 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -35,6 +35,9 @@ #if HAVE_SYS_EPOLL_H # include #endif +#ifdef HAVE_SCHED_H +# include +#endif #include "rsyslog.h" #include "dirty.h" #include "net.h" @@ -78,12 +81,103 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * termination if we can not get it. -- rgerhards, 2007-12-27 */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ +static uchar *pszSchedPolicy = NULL; /* scheduling policy string */ +static int iSchedPolicy; /* scheduling policy as SCHED_xxx */ +static int iSchedPrio; /* scheduling priority */ +static int seen_iSchedPrio = 0; /* have we seen scheduling priority in the config file? */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ /* config settings */ +static rsRetVal check_scheduling_priority(int report_error) +{ + DEFiRet; + +#ifdef HAVE_SCHED_GET_PRIORITY_MAX + if (iSchedPrio < sched_get_priority_min(iSchedPolicy) || + iSchedPrio > sched_get_priority_max(iSchedPolicy)) { + if (report_error) + errmsg.LogError(errno, NO_ERRCODE, + "imudp: scheduling priority %d out of range (%d - %d)" + " for scheduling policy '%s' - ignoring settings", + iSchedPrio, + sched_get_priority_min(iSchedPolicy), + sched_get_priority_max(iSchedPolicy), + pszSchedPolicy); + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } +#endif + +finalize_it: + RETiRet; +} + +/* Set scheduling priority in the supplied variable (will be iSchedPrio) + * and record that we have seen the directive (in seen_iSchedPrio). + */ +static rsRetVal set_scheduling_priority(void *pVal, int value) +{ + DEFiRet; + + if (seen_iSchedPrio) { + errmsg.LogError(0, NO_ERRCODE, "directive already seen"); + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } + *(int *)pVal = value; + seen_iSchedPrio = 1; + if (pszSchedPolicy != NULL) + CHKiRet(check_scheduling_priority(1)); + +finalize_it: + RETiRet; +} + +/* Set scheduling policy in iSchedPolicy */ +static rsRetVal set_scheduling_policy(void *pVal, uchar *pNewVal) +{ + int have_sched_policy = 0; + DEFiRet; + + if (pszSchedPolicy != NULL) { + errmsg.LogError(0, NO_ERRCODE, "directive already seen"); + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } + *((uchar**)pVal) = pNewVal; /* pVal is pszSchedPolicy */ + if (0) { /* trick to use conditional compilation */ +#ifdef SCHED_FIFO + } else if (!strcasecmp((char*)pszSchedPolicy, "fifo")) { + iSchedPolicy = SCHED_FIFO; + have_sched_policy = 1; +#endif +#ifdef SCHED_RR + } else if (!strcasecmp((char*)pszSchedPolicy, "rr")) { + iSchedPolicy = SCHED_RR; + have_sched_policy = 1; +#endif +#ifdef SCHED_OTHER + } else if (!strcasecmp((char*)pszSchedPolicy, "other")) { + iSchedPolicy = SCHED_OTHER; + have_sched_policy = 1; +#endif + } else { + errmsg.LogError(errno, NO_ERRCODE, + "imudp: invalid scheduling policy '%s' " + "- ignoring setting", pszSchedPolicy); + } + if (have_sched_policy == 0) { + free(pszSchedPolicy); + pszSchedPolicy = NULL; + ABORT_FINALIZE(RS_RET_VALIDATION_RUN); + } + if (seen_iSchedPrio) + CHKiRet(check_scheduling_priority(1)); + +finalize_it: + RETiRet; +} + /* This function is called when a new listener shall be added. It takes * the configured parameters, tries to bind the socket and, if that @@ -294,6 +388,41 @@ finalize_it: RETiRet; } +static void set_thread_schedparam(void) +{ + struct sched_param sparam; + + if (pszSchedPolicy != NULL && seen_iSchedPrio == 0) { + errmsg.LogError(0, NO_ERRCODE, + "imudp: scheduling policy set, but without priority - ignoring settings"); + } else if (pszSchedPolicy == NULL && seen_iSchedPrio != 0) { + errmsg.LogError(0, NO_ERRCODE, + "imudp: scheduling priority set, but without policy - ignoring settings"); + } else if (pszSchedPolicy != NULL && seen_iSchedPrio != 0 && + check_scheduling_priority(0) == 0) { +#ifndef HAVE_PTHREAD_SETSCHEDPARAM + errmsg.LogError(0, NO_ERRCODE, + "imudp: cannot set thread scheduling policy, " + "pthread_setschedparam() not available"); +#else + int err; + + memset(&sparam, 0, sizeof sparam); + sparam.sched_priority = iSchedPrio; + dbgprintf("imudp trying to set sched policy to '%s', prio %d\n", + pszSchedPolicy, iSchedPrio); + err = pthread_setschedparam(pthread_self(), iSchedPolicy, &sparam); + if (err != 0) { + errmsg.LogError(err, NO_ERRCODE, "imudp: pthread_setschedparam() failed"); + } +#endif + } + + if (pszSchedPolicy != NULL) { + free(pszSchedPolicy); + pszSchedPolicy = NULL; + } +} /* This function implements the main reception loop. Depending on the environment, * we either use the traditional (but slower) select() or the Linux-specific epoll() @@ -317,6 +446,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ + set_thread_schedparam(); bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); @@ -384,6 +514,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ + set_thread_schedparam(); bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); DBGPRINTF("imudp uses select()\n"); @@ -539,6 +670,10 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpolicy", 0, eCmdHdlrGetWord, + &set_scheduling_policy, &pszSchedPolicy, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imudpschedulingpriority", 0, eCmdHdlrInt, + &set_scheduling_priority, &iSchedPrio, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, -- cgit From 31125de72f3d787efb5b6ae8ba3117541b331bbc Mon Sep 17 00:00:00 2001 From: David Lang Date: Thu, 13 Jan 2011 09:08:33 +0100 Subject: added pmcisconames (uncompiled & untested) Signed-off-by: Rainer Gerhards --- plugins/pmcisconames/Makefile.am | 8 ++ plugins/pmcisconames/pmcisconames.c | 153 ++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 plugins/pmcisconames/Makefile.am create mode 100644 plugins/pmcisconames/pmcisconames.c (limited to 'plugins') diff --git a/plugins/pmcisconames/Makefile.am b/plugins/pmcisconames/Makefile.am new file mode 100644 index 00000000..16ed347d --- /dev/null +++ b/plugins/pmcisconames/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmcisconames.la + +pmcisconames_la_SOURCES = pmcisconames.c +pmcisconames_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmcisconames_la_LDFLAGS = -module -avoid-version +pmcisconames_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c new file mode 100644 index 00000000..28fe33d0 --- /dev/null +++ b/plugins/pmcisconames/pmcisconames.c @@ -0,0 +1,153 @@ +/* pmcisconames.c + * + * this detects logs sent by Cisco devices that mangle their syslog output when you tell them to log by name by adding ' :' between the name and the %XXX-X-XXXXXXX: tag + * + * instead of actually parsing the message, this modifies the message and then falls through to allow a later parser to handle the now modified message + * + * created 2010-12-13 by David Lang based on pmlastmsg + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.lastline") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINparse + uchar *p2parse; + int lenMsg; +#define OpeningText ": %" +CODESTARTparse + dbgprintf("Message will now be parsed by fix Cisco Names parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + + /* check if this message is of the type we handle in this (very limited) parser */ + /* first, we permit SP */ + while(lenMsg && *p2parse == ' ') { + --lenMsg; + ++p2parse; + } +dbgprintf("pmcisconames: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < 34) { + /* too short, can not be "our" message */ + /* minimum message, 16 character timestamp, 1 character name, ' : %ASA-1-000000: '*/ +dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* skip over timestamp */ + lenMsg -=16; + p2parse +=16; + /* now look for the next space to walk past the hostname */ + while(lenMsg && *p2parse != ' ') { + --lenMsg; + ++p2parse; + } + /* skip the space after the hostname */ + lenMsg -=1; + p2parse +=1; + /* if the syslog tag is : and the next thing starts with a % assume that this is a mangled cisco log and fix it */ + if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { + /* wrong opening text */ +dbgprintf("not a cisco name mangled log!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + /* bump the message portion up by two characters to overwrite the extra : */ + memmove(p2parse, p2parse + 2, lenMsg - 2); + /* now, claim to abort so that something else can parse the now modified message */ + DBGPRINTF("pmcisconames detected a mangled Cisco log message message\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ -- cgit From 33e5e60734f654256ac33781728777798bc88d1c Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 25 Jan 2011 14:13:14 +0100 Subject: pmciscomsg: fixed some problems Signed-off-by: Rainer Gerhards --- plugins/pmcisconames/pmcisconames.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'plugins') diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c index 28fe33d0..47d1f6f6 100644 --- a/plugins/pmcisconames/pmcisconames.c +++ b/plugins/pmcisconames/pmcisconames.c @@ -41,7 +41,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER -PARSER_NAME("rsyslog.lastline") +PARSER_NAME("rsyslog.cisconames") /* internal structures */ @@ -108,9 +108,14 @@ dbgprintf("not a cisco name mangled log!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } /* bump the message portion up by two characters to overwrite the extra : */ - memmove(p2parse, p2parse + 2, lenMsg - 2); + lenMsg -=2; + memmove(p2parse, p2parse + 2, lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=2; + pMsg->iLenMSG -=2; /* now, claim to abort so that something else can parse the now modified message */ - DBGPRINTF("pmcisconames detected a mangled Cisco log message message\n"); + DBGPRINTF("pmcisconames: new mesage: [%d]'%s'\n", lenMsg, p2parse); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); finalize_it: @@ -143,7 +148,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(parser, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); - dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION); + DBGPRINTF("cisconames parser init called, compiled with version %s\n", VERSION); bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ -- cgit From 9813a12789d08a00fa96c11f113cd531c7f2ce7b Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 25 Jan 2011 14:24:50 +0100 Subject: enhanced imfile to support multi-line messages Signed-off-by: Rainer Gerhards --- plugins/imfile/imfile.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 8681ac8c..c205f60e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -71,6 +71,7 @@ typedef struct fileInfo_s { int nRecords; /**< How many records did we process before persisting the stream? */ int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ + int readMode; /* which mode to use in ReadMulteLine call? */ } fileInfo_t; @@ -85,6 +86,7 @@ static int iPollInterval = 10; /* number of seconds to sleep when there was no f static int iPersistStateInterval = 0; /* how often if state file to be persisted? (default 0->never) */ static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ +static int readMode = 0; /* mode to use for ReadMultiLine call */ static int iFilPtr = 0; /* number of files to be monitored; pointer to next free spot during config */ #define MAX_INPUT_FILES 100 @@ -212,7 +214,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) /* loop below will be exited when strmReadLine() returns EOF */ while(1) { - CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr)); + CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr, pThis->readMode)); *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ @@ -447,6 +449,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iPollInterval = 10; iFacility = 128; /* local0 */ iSeverity = 5; /* notice, as of rfc 3164 */ + readMode = 0; RETiRet; } @@ -489,6 +492,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->iFacility = iFacility; pThis->iPersistStateInterval = iPersistStateInterval; pThis->nRecords = 0; + pThis->readMode = readMode; iPersistStateInterval = 0; } else { errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one"); @@ -535,6 +539,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepollinterval", 0, eCmdHdlrInt, NULL, &iPollInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilereadmode", 0, eCmdHdlrInt, + NULL, &readMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); /* that command ads a new file! */ -- cgit From 44e3fb96c5dd94a0669beff70cfa9d9097c35de2 Mon Sep 17 00:00:00 2001 From: Ariel P Date: Mon, 31 Jan 2011 17:22:39 +0100 Subject: added $OMMySQLConfigFile/$OMMySQLConfigSection config directives Signed-off-by: Rainer Gerhards --- plugins/ommysql/ommysql.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'plugins') diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index d6870a7b..aff76d0a 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -60,9 +60,13 @@ typedef struct _instanceData { char f_dbuid[_DB_MAXUNAMELEN+1]; /* DB user */ char f_dbpwd[_DB_MAXPWDLEN+1]; /* DB user's password */ unsigned uLastMySQLErrno; /* last errno returned by MySQL or 0 if all is well */ + uchar * f_configfile; /* MySQL Client Configuration File */ + uchar * f_configsection; /* MySQL Client Configuration Section */ } instanceData; /* config variables */ +static uchar * pszMySQLConfigFile = NULL; /* MySQL Client Configuration File */ +static uchar * pszMySQLConfigSection = NULL; /* MySQL Client Configuration Section */ static int iSrvPort = 0; /* database server port */ @@ -91,6 +95,14 @@ static void closeMySQL(instanceData *pData) mysql_close(pData->f_hmysql); pData->f_hmysql = NULL; } + if(pData->f_configfile!=NULL){ + free(pData->f_configfile); + pData->f_configfile=NULL; + } + if(pData->f_configsection!=NULL){ + free(pData->f_configsection); + pData->f_configsection=NULL; + } } BEGINfreeInstance @@ -152,6 +164,25 @@ static rsRetVal initMySQL(instanceData *pData, int bSilent) errmsg.LogError(0, RS_RET_SUSPENDED, "can not initialize MySQL handle"); iRet = RS_RET_SUSPENDED; } else { /* we could get the handle, now on with work... */ + mysql_options(pData->f_hmysql,MYSQL_READ_DEFAULT_GROUP,((pData->f_configsection!=NULL)?(char*)pData->f_configsection:"client")); + if(pData->f_configfile!=NULL){ + FILE * fp; + fp=fopen((char*)pData->f_configfile,"r"); + int err=errno; + if(fp==NULL){ + char msg[512]; + snprintf(msg,sizeof(msg)/sizeof(char),"Could not open '%s' for reading",pData->f_configfile); + if(bSilent) { + char errStr[512]; + rs_strerror_r(err, errStr, sizeof(errStr)); + dbgprintf("mysql configuration error(%d): %s - %s\n",err,msg,errStr); + } else + errmsg.LogError(err,NO_ERRCODE,"mysql configuration error: %s\n",msg); + } else { + fclose(fp); + mysql_options(pData->f_hmysql,MYSQL_READ_DEFAULT_FILE,pData->f_configfile); + } + } /* Connect to database */ if(mysql_real_connect(pData->f_hmysql, pData->f_dbsrv, pData->f_dbuid, pData->f_dbpwd, pData->f_dbname, pData->f_dbsrvPort, NULL, 0) == NULL) { @@ -278,6 +309,8 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } else { pData->f_dbsrvPort = (unsigned) iSrvPort; /* set configured port */ + pData->f_configfile = pszMySQLConfigFile; + pData->f_configsection = pszMySQLConfigSection; pData->f_hmysql = NULL; /* initialize, but connect only on first message (important for queued mode!) */ } @@ -302,6 +335,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a { DEFiRet; iSrvPort = 0; /* zero is the default port */ + free(pszMySQLConfigFile); + pszMySQLConfigFile = NULL; + free(pszMySQLConfigSection); + pszMySQLConfigSection = NULL; RETiRet; } @@ -313,6 +350,8 @@ CODEmodInit_QueryRegCFSLineHdlr /* register our config handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionommysqlserverport", 0, eCmdHdlrInt, NULL, &iSrvPort, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"ommysqlconfigfile",0,eCmdHdlrGetWord,NULL,&pszMySQLConfigFile,STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"ommysqlconfigsection",0,eCmdHdlrGetWord,NULL,&pszMySQLConfigSection,STD_LOADABLE_MODULE_ID)); ENDmodInit /* vi:set ai: -- cgit From 58cb4d57e81da5aa74ed2927f99b576cec83d71b Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 1 Feb 2011 11:36:27 +0100 Subject: added pmaixforwardedfrom message parser Signed-off-by: Rainer Gerhards --- plugins/pmaixforwardedfrom/Makefile.am | 8 ++ plugins/pmaixforwardedfrom/pmaixforwardedfrom.c | 167 ++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 plugins/pmaixforwardedfrom/Makefile.am create mode 100644 plugins/pmaixforwardedfrom/pmaixforwardedfrom.c (limited to 'plugins') diff --git a/plugins/pmaixforwardedfrom/Makefile.am b/plugins/pmaixforwardedfrom/Makefile.am new file mode 100644 index 00000000..af359d31 --- /dev/null +++ b/plugins/pmaixforwardedfrom/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmaixforwardedfrom.la + +pmaixforwardedfrom_la_SOURCES = pmaixforwardedfrom.c +pmaixforwardedfrom_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmaixforwardedfrom_la_LDFLAGS = -module -avoid-version +pmaixforwardedfrom_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c new file mode 100644 index 00000000..11634199 --- /dev/null +++ b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c @@ -0,0 +1,167 @@ +/* pmaixforwardedfrom.c + * + * this detects logs sent by Cisco devices that mangle their syslog output when you tell them to log by name by adding ' :' between the name and the %XXX-X-XXXXXXX: tag + * + * instead of actually parsing the message, this modifies the message and then falls through to allow a later parser to handle the now modified message + * + * created 2010-12-13 by David Lang based on pmlastmsg + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.aixforwardedfrom") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINparse + uchar *p2parse; + uchar *opening; + int lenMsg; +#define OpeningText "Message forwarded from " +CODESTARTparse + dbgprintf("Message will now be parsed by fix AIX Forwarded From parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + + /* check if this message is of the type we handle in this (very limited) parser */ + /* first, we permit SP */ + while(lenMsg && *p2parse == ' ') { + --lenMsg; + ++p2parse; + } +dbgprintf("pmaixforwardedfrom: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < 42) { + /* too short, can not be "our" message */ + /* minimum message, 16 character timestamp, 'Message forwarded from ", 1 character name, ': '*/ +dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* skip over timestamp */ + lenMsg -=16; + p2parse +=16; + /* if there is the string "Message forwarded from " were the hostname should be */ + if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { + /* wrong opening text */ +dbgprintf("not a AIX message forwarded from mangled log!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + /* bump the message portion up by 23 characters to overwrite the "Message forwarded from " with the hostname */ + lenMsg -=23; + memmove(p2parse, p2parse + 23, lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=23; + pMsg->iLenMSG -=23; + /* now look for the : after the hostname to walk past the hostname, also watch for a space in case this isn't really an AIX log, but has a similar preamble */ + while(lenMsg && *p2parse != ' ' && *p2parse != ':') { + --lenMsg; + ++p2parse; + } + if (lenMsg && *p2parse != ':') { +dbgprintf("not a AIX message forwarded from mangled log but similar enough that the preamble has been removed\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + /* bump the message portion up by one character to overwrite the extra : */ + lenMsg -=1; + memmove(p2parse, p2parse + 1, lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=1; + pMsg->iLenMSG -=1; + /* now, claim to abort so that something else can parse the now modified message */ + DBGPRINTF("pmaixforwardedfrom: new mesage: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + DBGPRINTF("aixforwardedfrom parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ -- cgit From fa89eef48e25608585454915ff7af9fae524042e Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 8 Feb 2011 11:46:05 +0100 Subject: added pmsnare parser module Signed-off-by: Rainer Gerhards --- plugins/pmsnare/Makefile.am | 8 ++ plugins/pmsnare/pmsnare.c | 233 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 plugins/pmsnare/Makefile.am create mode 100644 plugins/pmsnare/pmsnare.c (limited to 'plugins') diff --git a/plugins/pmsnare/Makefile.am b/plugins/pmsnare/Makefile.am new file mode 100644 index 00000000..5b2696ac --- /dev/null +++ b/plugins/pmsnare/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmsnare.la + +pmsnare_la_SOURCES = pmsnare.c +pmsnare_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmsnare_la_LDFLAGS = -module -avoid-version +pmsnare_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmsnare/pmsnare.c b/plugins/pmsnare/pmsnare.c new file mode 100644 index 00000000..ce2138b2 --- /dev/null +++ b/plugins/pmsnare/pmsnare.c @@ -0,0 +1,233 @@ +/* pmsnare.c + * + * this detects logs sent by Snare and cleans them up so that they can be processed by the normal parser + * + * there are two variations of this, if the client is set to 'syslog' mode it sends + * + * timestamphostnametagotherstuff + * + * if the client is not set to syslog it sends + * + * hostnametagotherstuff + * + * ToDo, take advantage of items in the message itself to set more friendly information + * where the normal parser will find it by re-writing more of the message + * + * Intereting information includes: + * + * in the case of windows snare messages: + * the system hostname is field 12 + * the severity is field 3 (criticality ranging form 0 to 4) + * the source of the log is field 4 and may be able to be mapped to facility + * + * + * created 2010-12-13 by David Lang based on pmlastmsg + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.snare") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINparse + uchar *p2parse; + int lenMsg; + int snaremessage; + int tablength; + +CODESTARTparse + #define TabRepresentation "#011" + tablength=sizeof(TabRepresentation); + dbgprintf("Message will now be parsed by fix Snare parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + + /* check if this message is of the type we handle in this (very limited) parser + + find out if we have a space separated or tab separated for the first item + if tab separated see if the second word is one of our expected tags + if so replace the tabs with spaces so that hostname and syslog tag are going to be parsed properly + optionally replace the hostname at the beginning of the message with one from later in the message + else, wrong message, abort + else, assume that we have a valid timestamp, move over to the syslog tag + if that is tab separated from the rest of the message and one of our expected tags + if so, replace the tab with a space so that it will be parsed properly + optionally replace the hostname at the beginning of the message withone from later in the message + + */ + snaremessage=0; + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + dbgprintf("pmsnare: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < 30) { + /* too short, can not be "our" message */ + dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + while(lenMsg && *p2parse != ' ' && *p2parse != '\t' && *p2parse != '#') { + --lenMsg; + ++p2parse; + } + dbgprintf("pmsnare: separator [%d]'%s' msg after the first separator: [%d]'%s'\n", tablength,TabRepresentation,lenMsg, p2parse); + if ((lenMsg > tablength) && (*p2parse == '\t' || strncasecmp((char*) p2parse, TabRepresentation , tablength-1) == 0)) { + //if ((lenMsg > tablength) && (*p2parse == '\t' || *p2parse == '#')) { + dbgprintf("pmsnare: tab separated message\n"); + if(strncasecmp((char*) (p2parse + tablength - 1), "MSWinEventLog", 13) == 0) { + snaremessage=13; /* 0 means not a snare message, a number is how long the tag is */ + } + if(strncasecmp((char*) (p2parse + tablength - 1), "LinuxKAudit", 11) == 0) { + snaremessage=11; /* 0 means not a snare message, a number is how long the tag is */ + } + if(snaremessage) { + /* replace the tab with a space and if needed move the message portion up by the length of TabRepresentation -2 characters to overwrite the extra : */ + *p2parse = ' '; + lenMsg -=(tablength-2); + p2parse++; + memmove(p2parse, p2parse + (tablength-2), lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=(tablength-2); + pMsg->iLenMSG -=(tablength-2); + p2parse += snaremessage; + *p2parse = ' '; + p2parse++; + lenMsg -=(tablength-2); + memmove(p2parse, p2parse + (tablength-2), lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=(tablength-2); + pMsg->iLenMSG -=(tablength-2); + dbgprintf("found a Snare message with snare not set to send syslog messages\n"); + } + } else { + /* go back to the beginning of the message */ + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + /* skip over timestamp and space*/ + lenMsg -=17; + p2parse +=17; + /* skip over what should be the hostname */ + while(lenMsg && *p2parse != ' ') { + --lenMsg; + ++p2parse; + } + if (lenMsg){ + --lenMsg; + ++p2parse; + } + dbgprintf("pmsnare: separator [%d]'%s' msg after the timestamp and hostname: [%d]'%s'\n", tablength,TabRepresentation,lenMsg, p2parse); + if(lenMsg > 13 && strncasecmp((char*) p2parse, "MSWinEventLog", 13) == 0) { + snaremessage=13; /* 0 means not a snare message, a number is how long the tag is */ + } + if(lenMsg > 11 && strncasecmp((char*) p2parse, "LinuxKAudit", 11) == 0) { + snaremessage=11; /* 0 means not a snare message, a number is how long the tag is */ + } + if(snaremessage) { + p2parse += snaremessage; + *p2parse = ' '; + p2parse++; + lenMsg -=(tablength-2); + memmove(p2parse, p2parse + (tablength-2), lenMsg); + *(p2parse + lenMsg) = '\n'; + *(p2parse + lenMsg + 1) = '\0'; + pMsg->iLenRawMsg -=(tablength-2); + pMsg->iLenMSG -=(tablength-2); + dbgprintf("found a Snare message with snare set to send syslog messages\n"); + } + + } + DBGPRINTF("pmsnare: new mesage: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + DBGPRINTF("snare parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ -- cgit From fce5fe5127c4b2e4ba17a64b8fb9d57b573baec0 Mon Sep 17 00:00:00 2001 From: David Lang Date: Thu, 10 Feb 2011 07:36:34 +0100 Subject: pmcisconames: bugfix for short timestamps Signed-off-by: Rainer Gerhards --- plugins/pmcisconames/pmcisconames.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) (limited to 'plugins') diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c index 47d1f6f6..4171e688 100644 --- a/plugins/pmcisconames/pmcisconames.c +++ b/plugins/pmcisconames/pmcisconames.c @@ -89,10 +89,29 @@ dbgprintf("pmcisconames: msg to look at: [%d]'%s'\n", lenMsg, p2parse); dbgprintf("msg too short!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } - - /* skip over timestamp */ - lenMsg -=16; - p2parse +=16; + /* check if the timestamp is a 16 character or 21 character timestamp + 'Mmm DD HH:MM:SS ' spaces at 3,6,15 : at 9,12 + 'Mmm DD YYYY HH:MM:SS ' spaces at 3,6,11,20 : at 14,17 + check for the : first as that will differentiate the two conditions the fastest + this allows the compiler to short circuit the rst of the tests if it is the wrong timestamp + but still check the rest to see if it looks correct + */ + if ( *(p2parse + 9) == ':' && *(p2parse + 12) == ':' && *(p2parse + 3) == ' ' && *(p2parse + 6) == ' ' && *(p2parse + 15) == ' ') { + /* skip over timestamp */ + dbgprintf("short timestamp found\n"); + lenMsg -=16; + p2parse +=16; + } else { + if ( *(p2parse + 14) == ':' && *(p2parse + 17) == ':' && *(p2parse + 3) == ' ' && *(p2parse + 6) == ' ' && *(p2parse + 11) == ' ' && *(p2parse + 20) == ' ') { + /* skip over timestamp */ + dbgprintf("long timestamp found\n"); + lenMsg -=21; + p2parse +=21; + } else { + dbgprintf("timestamp is not one of the valid formats\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + } /* now look for the next space to walk past the hostname */ while(lenMsg && *p2parse != ' ') { --lenMsg; -- cgit From 2c1021a01f75f5bb3e16b7b952a2d29494943bed Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Feb 2011 18:23:34 +0100 Subject: enhanced imfile to support non-cancel input termination --- plugins/imfile/imfile.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'plugins') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index c205f60e..7f6b9c24 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -213,7 +213,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) } /* loop below will be exited when strmReadLine() returns EOF */ - while(1) { + while(glbl.GetGlobalInputTermState() == 0) { CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr, pThis->readMode)); *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ @@ -289,9 +289,10 @@ BEGINrunInput int bHadFileData; /* were there at least one file with data during this run? */ CODESTARTrunInput pthread_cleanup_push(inputModuleCleanup, NULL); - while(1) { - + while(glbl.GetGlobalInputTermState() == 0) { do { + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ bHadFileData = 0; for(i = 0 ; i < iFilPtr ; ++i) { pollFile(&files[i], &bHadFileData); @@ -302,10 +303,10 @@ CODESTARTrunInput * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any * other valid scenario. So do not remove. -- rgerhards, 2008-02-14 */ - srSleep(iPollInterval, 10); - + if(glbl.GetGlobalInputTermState() == 0) + srSleep(iPollInterval, 10); } - /*NOTREACHED*/ + DBGPRINTF("imfile: terminating upon request of rsyslog core\n"); pthread_cleanup_pop(0); /* just for completeness, but never called... */ RETiRet; /* use it to make sure the housekeeping is done! */ @@ -398,6 +399,13 @@ CODESTARTafterRun ENDafterRun +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + /* The following entry points are defined in module-template.h. * In general, they need to be present, but you do NOT need to provide * any code here. @@ -416,6 +424,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt @@ -527,6 +536,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + DBGPRINTF("imfile: version %s initializing\n", VERSION); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilename", 0, eCmdHdlrGetWord, NULL, &pszFileName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfiletag", 0, eCmdHdlrGetWord, -- cgit From a9736c6215cf6446c326b034648f63a79c0426f9 Mon Sep 17 00:00:00 2001 From: David Lang Date: Fri, 11 Feb 2011 07:44:09 +0100 Subject: bugfix in pmsnare when removing text from the message, decrement the length when you increment the pointer through the message. there were a number of places where this was missed. Signed-off-by: Rainer Gerhards --- plugins/pmsnare/pmsnare.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/pmsnare/pmsnare.c b/plugins/pmsnare/pmsnare.c index ce2138b2..4a9880d4 100644 --- a/plugins/pmsnare/pmsnare.c +++ b/plugins/pmsnare/pmsnare.c @@ -137,14 +137,17 @@ CODESTARTparse *p2parse = ' '; lenMsg -=(tablength-2); p2parse++; + lenMsg--; memmove(p2parse, p2parse + (tablength-2), lenMsg); *(p2parse + lenMsg) = '\n'; *(p2parse + lenMsg + 1) = '\0'; pMsg->iLenRawMsg -=(tablength-2); pMsg->iLenMSG -=(tablength-2); p2parse += snaremessage; + lenMsg -= snaremessage; *p2parse = ' '; p2parse++; + lenMsg--; lenMsg -=(tablength-2); memmove(p2parse, p2parse + (tablength-2), lenMsg); *(p2parse + lenMsg) = '\n'; @@ -178,8 +181,10 @@ CODESTARTparse } if(snaremessage) { p2parse += snaremessage; + lenMsg -= snaremessage; *p2parse = ' '; p2parse++; + lenMsg--; lenMsg -=(tablength-2); memmove(p2parse, p2parse + (tablength-2), lenMsg); *(p2parse + lenMsg) = '\n'; @@ -190,7 +195,7 @@ CODESTARTparse } } - DBGPRINTF("pmsnare: new mesage: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + DBGPRINTF("pmsnare: new message: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); finalize_it: -- cgit From 1b84d5998926df6ef1b2bbfc9fe5a59a80c5564c Mon Sep 17 00:00:00 2001 From: Marius Tomaschewski Date: Fri, 11 Feb 2011 09:24:45 +0100 Subject: Improved systemd socket activation support Support for multiple unix sockets and activation in forking mode Signed-off-by: Marius Tomaschewski --- plugins/imuxsock/imuxsock.c | 89 +++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 44 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 0eee1122..e456a113 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -143,7 +143,7 @@ static int startIndexUxLocalSockets; /* process fd from that index on (used to * read-only after startup */ static int nfd = 1; /* number of Unix sockets open / read-only after startup */ -static int bSysSockFromSystemd = 0; /* Did we receive the system socket from systemd? */ +static int sd_fds = 0; /* number of systemd activated sockets */ /* config settings */ static int bOmitLocalLogging = 0; @@ -372,41 +372,32 @@ openLogSocket(lstn_t *pLstn) if(pLstn->sockName[0] == '\0') return -1; - if (ustrcmp(pLstn->sockName, UCHAR_CONSTANT(_PATH_LOG)) == 0) { - bSysSockFromSystemd = 0; /* set default */ - int r; - - /* System log socket code. Check whether an FD was passed in from systemd. If - * so, it's the /dev/log socket, so use it. */ - - r = sd_listen_fds(0); - if (r < 0) { - errmsg.LogError(-r, NO_ERRCODE, "Failed to acquire systemd socket"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - - if (r > 1) { - errmsg.LogError(EINVAL, NO_ERRCODE, "Wrong number of systemd sockets passed"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - - if (r == 1) { - pLstn->fd = SD_LISTEN_FDS_START; - r = sd_is_socket_unix(pLstn->fd, SOCK_DGRAM, -1, _PATH_LOG, 0); - if (r < 0) { - errmsg.LogError(-r, NO_ERRCODE, "Failed to verify systemd socket type"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - - if (!r) { - errmsg.LogError(EINVAL, NO_ERRCODE, "Passed systemd socket of wrong type"); - ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); - } - bSysSockFromSystemd = 1; /* indicate we got the socket from systemd */ - } else { - CHKiRet(createLogSocket(pLstn)); + pLstn->fd = -1; + + if (sd_fds > 0) { + /* Check if the current socket is a systemd activated one. + * If so, just use it. + */ + int fd; + + for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + sd_fds; fd++) { + if( sd_is_socket_unix(fd, SOCK_DGRAM, -1, pLstn->sockName, 0) == 1) { + /* ok, it matches -- just use as is */ + pLstn->fd = fd; + + dbgprintf("imuxsock: Acquired UNIX socket '%s' (fd %d) from systemd.\n", + pLstn->sockName, pLstn->fd); + break; + } + /* + * otherwise it either didn't matched *this* socket and + * we just continue to check the next one or there were + * an error and we will create a new socket bellow. + */ } - } else { + } + + if (pLstn->fd == -1) { CHKiRet(createLogSocket(pLstn)); } @@ -774,12 +765,18 @@ CODESTARTwillRun listeners[0].bUseCreds = (bWritePidSysSock || ratelimitIntervalSysSock) ? 1 : 0; listeners[0].bWritePid = bWritePidSysSock; + sd_fds = sd_listen_fds(0); + if (sd_fds < 0) { + errmsg.LogError(-r, NO_ERRCODE, "imuxsock: Failed to acquire systemd socket"); + ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); + } + /* initialize and return if will run or not */ actSocks = 0; for (i = startIndexUxLocalSockets ; i < nfd ; i++) { if(openLogSocket(&(listeners[i])) == RS_RET_OK) { ++actSocks; - dbgprintf("Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); + dbgprintf("imuxsock: Opened UNIX socket '%s' (fd %d).\n", listeners[i].sockName, listeners[i].fd); } } @@ -806,15 +803,19 @@ CODESTARTafterRun if (listeners[i].fd != -1) close(listeners[i].fd); - /* Clean-up files. If systemd passed us a socket it is - * systemd's job to clean it up.*/ - if(bSysSockFromSystemd) { - DBGPRINTF("imuxsock: got system socket from systemd, not unlinking it\n"); - i = 1; - } else - i = startIndexUxLocalSockets; - for(; i < nfd; i++) + /* Clean-up files. */ + for(i = startIndexUxLocalSockets; i < nfd; i++) if (listeners[i].sockName && listeners[i].fd != -1) { + + /* If systemd passed us a socket it is systemd's job to clean it up. + * Do not unlink it -- we will get same socket (node) from systemd + * e.g. on restart again. + */ + if (sd_fds > 0 && + listeners[i].fd >= SD_LISTEN_FDS_START && + listeners[i].fd < SD_LISTEN_FDS_START + sd_fds) + continue; + DBGPRINTF("imuxsock: unlinking unix socket file[%d] %s\n", i, listeners[i].sockName); unlink((char*) listeners[i].sockName); } -- cgit From c90079aaf07ed574b11ade34af899cb4980ac9f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 11:25:06 +0100 Subject: fixed some compile issues in recent systemd patch --- plugins/imuxsock/imuxsock.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index e456a113..75f97db4 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -381,7 +381,7 @@ openLogSocket(lstn_t *pLstn) int fd; for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + sd_fds; fd++) { - if( sd_is_socket_unix(fd, SOCK_DGRAM, -1, pLstn->sockName, 0) == 1) { + if( sd_is_socket_unix(fd, SOCK_DGRAM, -1, (const char*) pLstn->sockName, 0) == 1) { /* ok, it matches -- just use as is */ pLstn->fd = fd; @@ -767,7 +767,7 @@ CODESTARTwillRun sd_fds = sd_listen_fds(0); if (sd_fds < 0) { - errmsg.LogError(-r, NO_ERRCODE, "imuxsock: Failed to acquire systemd socket"); + errmsg.LogError(-sd_fds, NO_ERRCODE, "imuxsock: Failed to acquire systemd socket"); ABORT_FINALIZE(RS_RET_ERR_CRE_AFUX); } -- cgit From f3d354da3e373f9c4890a78e5274a6ba02f1c8cb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 17:47:30 +0100 Subject: bugfix: very long running actions could prevent shutdown under some circumstances This has now been solved, at least for common situations. --- plugins/imfile/imfile.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 7f6b9c24..c71e425e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -291,13 +291,13 @@ CODESTARTrunInput pthread_cleanup_push(inputModuleCleanup, NULL); while(glbl.GetGlobalInputTermState() == 0) { do { - if(glbl.GetGlobalInputTermState() == 1) - break; /* terminate input! */ bHadFileData = 0; for(i = 0 ; i < iFilPtr ; ++i) { + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ pollFile(&files[i], &bHadFileData); } - } while(iFilPtr > 1 && bHadFileData == 1); /* warning: do...while()! */ + } while(iFilPtr > 1 && bHadFileData == 1 && glbl.GetGlobalInputTermState() == 0); /* warning: do...while()! */ /* Note: the additional 10ns wait is vitally important. It guards rsyslog against totally * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any -- cgit From 2f1ef6164afa3a91f0bdd92507d0e8d557bea555 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 18 Feb 2011 16:04:50 +0100 Subject: enhance: imfile did not yet support multiple rulesets, now added we do this directly in the beta because a) it does not affect existing functionality and b) one may argue that this missing functionality is close to a bug. --- plugins/imfile/imfile.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'plugins') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index c71e425e..1dd5e65c 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -48,6 +48,7 @@ #include "unicode-helper.h" #include "prop.h" #include "stringbuf.h" +#include "ruleset.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -60,6 +61,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) DEFobjCurrIf(strm) DEFobjCurrIf(prop) +DEFobjCurrIf(ruleset) typedef struct fileInfo_s { uchar *pszFileName; @@ -72,6 +74,7 @@ typedef struct fileInfo_s { int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ int readMode; /* which mode to use in ReadMulteLine call? */ + ruleset_t *pRuleset; /* ruleset to bind listener to (use system default if unspecified) */ } fileInfo_t; @@ -87,6 +90,7 @@ static int iPersistStateInterval = 0; /* how often if state file to be persisted static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ static int readMode = 0; /* mode to use for ReadMultiLine call */ +static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ static int iFilPtr = 0; /* number of files to be monitored; pointer to next free spot during config */ #define MAX_INPUT_FILES 100 @@ -116,6 +120,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); + MsgSetRuleset(pMsg, pInfo->pRuleset); CHKiRet(submitMsg(pMsg)); finalize_it: RETiRet; @@ -418,6 +423,7 @@ CODESTARTmodExit objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDmodExit @@ -459,6 +465,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iFacility = 128; /* local0 */ iSeverity = 5; /* notice, as of rfc 3164 */ readMode = 0; + pBindRuleset = NULL; RETiRet; } @@ -502,6 +509,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->iPersistStateInterval = iPersistStateInterval; pThis->nRecords = 0; pThis->readMode = readMode; + pThis->pRuleset = pBindRuleset; iPersistStateInterval = 0; } else { errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one"); @@ -517,6 +525,29 @@ finalize_it: RETiRet; } + +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal +setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imfile current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + /* modInit() is called once the module is loaded. It must perform all module-wide * initialization tasks. There are also a number of housekeeping tasks that the * framework requires. These are handled by the macros. Please note that the @@ -534,6 +565,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); DBGPRINTF("imfile: version %s initializing\n", VERSION); @@ -553,6 +585,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &readMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilebindruleset", 0, eCmdHdlrGetWord, + setRuleset, NULL, STD_LOADABLE_MODULE_ID)); /* that command ads a new file! */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrunfilemonitor", 0, eCmdHdlrGetWord, addMonitor, NULL, STD_LOADABLE_MODULE_ID)); -- cgit From 026f59388a105597c3c6890bb0c83652cb712903 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Feb 2011 09:10:45 +0100 Subject: bugfix: imuxsock did no longer remove trailing LF This was a regression from the imuxsock partial rewrite. Happened because the message is no longer run through the standard parsers. Now imuxsock does this sanitization itself. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=224 --- plugins/imuxsock/imuxsock.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 75f97db4..136e4196 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -517,6 +517,10 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) * rate-limiting as well. */ parse = pRcv; + if(pRcv[lenRcv-1] = '\n') { + DBGPRINTF("imuxsock: found trailing LF, removing\n"); + lenRcv--; + } lenMsg = lenRcv; parse++; lenMsg--; /* '<' */ -- cgit From 314d3532109cca574aabc3e960586f499966e868 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Feb 2011 11:41:50 +0100 Subject: fixed dumb but important typo in code... thanks to Michael Biebl for alterting me --- plugins/imuxsock/imuxsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 136e4196..0b5ca6be 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -517,7 +517,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) * rate-limiting as well. */ parse = pRcv; - if(pRcv[lenRcv-1] = '\n') { + if(pRcv[lenRcv-1] == '\n') { DBGPRINTF("imuxsock: found trailing LF, removing\n"); lenRcv--; } -- cgit From 70bd55d51b6107231b841bfa03d5620bf984e3f2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Feb 2011 11:55:12 +0100 Subject: bugfix: imuxsock does not sanitization at all (regression) This also causes problems when NUL characters are present inside the message (as is the case with spamd). --- plugins/imuxsock/imuxsock.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 0b5ca6be..9f1e51b7 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -46,6 +46,7 @@ #include "net.h" #include "glbl.h" #include "msg.h" +#include "parser.h" #include "prop.h" #include "debug.h" #include "unlimited_select.h" @@ -81,6 +82,7 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) +DEFobjCurrIf(parser) DEFobjCurrIf(datetime) DEFobjCurrIf(statsobj) @@ -517,10 +519,6 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) * rate-limiting as well. */ parse = pRcv; - if(pRcv[lenRcv-1] == '\n') { - DBGPRINTF("imuxsock: found trailing LF, removing\n"); - lenRcv--; - } lenMsg = lenRcv; parse++; lenMsg--; /* '<' */ @@ -547,6 +545,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); + parser.SanitizeMsg(pMsg); MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); @@ -840,6 +839,7 @@ BEGINmodExit CODESTARTmodExit statsobj.Destruct(&modStats); + objRelease(parser, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); @@ -901,6 +901,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(statsobj, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); -- cgit From b7ee1de6b0dbdc67bbb239f44719fb4a50054fb5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Feb 2011 15:35:52 +0100 Subject: the last fix introduced another regression, fixed now The previous fix left variable lenMsg in an inconsistent state when sanitization actually happend. This could lead to message truncation. --- plugins/imuxsock/imuxsock.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 9f1e51b7..ff38852c 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -503,6 +503,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) { msg_t *pMsg; int lenMsg; + int offs; int i; uchar *parse; int pri; @@ -520,13 +521,14 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) */ parse = pRcv; lenMsg = lenRcv; + offs = 1; /* '<' */ - parse++; lenMsg--; /* '<' */ + parse++; pri = 0; - while(lenMsg && isdigit(*parse)) { + while(offs < lenMsg && isdigit(*parse)) { pri = pri * 10 + *parse - '0'; ++parse; - --lenMsg; + ++offs; } facil = LOG_FAC(pri); sever = LOG_PRI(pri); @@ -546,12 +548,13 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); parser.SanitizeMsg(pMsg); + lenMsg = pMsg->iLenRawMsg - offs; MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, pLstn->flowCtl); pMsg->iFacility = facil; pMsg->iSeverity = sever; - MsgSetAfterPRIOffs(pMsg, lenRcv - lenMsg); + MsgSetAfterPRIOffs(pMsg, offs); parse++; lenMsg--; /* '>' */ @@ -571,7 +574,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) fixPID(bufParseTAG, &i, cred); MsgSetTAG(pMsg, bufParseTAG, i); - MsgSetMSGoffs(pMsg, lenRcv - lenMsg); + MsgSetMSGoffs(pMsg, pMsg->iLenRawMsg - lenMsg); if(pLstn->bParseHost) { pMsg->msgFlags = pLstn->flags | PARSE_HOSTNAME; -- cgit From d1eb6e0edc51a78f3209448e800b25eda50340f2 Mon Sep 17 00:00:00 2001 From: Bojan Smojver Date: Wed, 23 Feb 2011 11:25:43 +0100 Subject: added work-around for bug in gtls, which causes fd leak when using TLS The capability has been added for module to specify that they do not like being unloaded. related bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=222 Signed-off-by: Rainer Gerhards --- plugins/im3195/im3195.c | 1 + plugins/imdiag/imdiag.c | 1 + plugins/imfile/imfile.c | 1 + plugins/imgssapi/imgssapi.c | 1 + plugins/imklog/imklog.c | 1 + plugins/immark/immark.c | 1 + plugins/imptcp/imptcp.c | 1 + plugins/imrelp/imrelp.c | 1 + plugins/imsolaris/imsolaris.c | 1 + plugins/imtcp/imtcp.c | 1 + plugins/imtemplate/imtemplate.c | 1 + plugins/imudp/imudp.c | 1 + plugins/imuxsock/imuxsock.c | 1 + plugins/omdbalerting/omdbalerting.c | 1 + plugins/omgssapi/omgssapi.c | 1 + plugins/omlibdbi/omlibdbi.c | 1 + plugins/ommail/ommail.c | 1 + plugins/ommysql/ommysql.c | 1 + plugins/omoracle/omoracle.c | 1 + plugins/ompgsql/ompgsql.c | 1 + plugins/omprog/omprog.c | 1 + plugins/omrelp/omrelp.c | 1 + plugins/omruleset/omruleset.c | 1 + plugins/omsnmp/omsnmp.c | 1 + plugins/omstdout/omstdout.c | 1 + plugins/omtemplate/omtemplate.c | 1 + plugins/omtesting/omtesting.c | 1 + plugins/omudpspoof/omudpspoof.c | 1 + plugins/omuxsock/omuxsock.c | 1 + plugins/pmlastmsg/pmlastmsg.c | 1 + plugins/pmrfc3164sd/pmrfc3164sd.c | 1 + 31 files changed, 31 insertions(+) (limited to 'plugins') diff --git a/plugins/im3195/im3195.c b/plugins/im3195/im3195.c index 106da2c8..156524c9 100644 --- a/plugins/im3195/im3195.c +++ b/plugins/im3195/im3195.c @@ -51,6 +51,7 @@ #include "errmsg.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* Module static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 81b357ef..b7a2a070 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -57,6 +57,7 @@ #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 1dd5e65c..acb58dad 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -51,6 +51,7 @@ #include "ruleset.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ +MODULE_TYPE_NOKEEP /* defines */ diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index dd3d67e3..446795d6 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -62,6 +62,7 @@ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define ALLOWEDMETHOD_GSS 2 diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index c59ce04f..69c8cd1a 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -58,6 +58,7 @@ #include "unicode-helper.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* Module static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/immark/immark.c b/plugins/immark/immark.c index 5d48369e..6410003d 100644 --- a/plugins/immark/immark.c +++ b/plugins/immark/immark.c @@ -46,6 +46,7 @@ #include "glbl.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define DEFAULT_MARK_PERIOD (20 * 60) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 6449ad62..3197564e 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -73,6 +73,7 @@ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 9be38f8f..13fd4428 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -47,6 +47,7 @@ #include "prop.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imsolaris/imsolaris.c b/plugins/imsolaris/imsolaris.c index f801833b..ee9ec5c6 100644 --- a/plugins/imsolaris/imsolaris.c +++ b/plugins/imsolaris/imsolaris.c @@ -85,6 +85,7 @@ #include "sun_cddl.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define PATH_LOG "/dev/log" diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 0cfae057..d3e9cabe 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -65,6 +65,7 @@ #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* static data */ DEF_IMOD_STATIC_DATA diff --git a/plugins/imtemplate/imtemplate.c b/plugins/imtemplate/imtemplate.c index e5e43025..0e2cac11 100644 --- a/plugins/imtemplate/imtemplate.c +++ b/plugins/imtemplate/imtemplate.c @@ -80,6 +80,7 @@ #include "debug.h" /* some debug helper functions */ MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ +MODULE_TYPE_NOKEEP /* defines */ diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index ad39ead0..56cdab28 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -54,6 +54,7 @@ #include "unicode-helper.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index ff38852c..40f32e4a 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -56,6 +56,7 @@ #include "hashtable.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define MAXFUNIX 50 diff --git a/plugins/omdbalerting/omdbalerting.c b/plugins/omdbalerting/omdbalerting.c index 2e04391c..35de5818 100644 --- a/plugins/omdbalerting/omdbalerting.c +++ b/plugins/omdbalerting/omdbalerting.c @@ -44,6 +44,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 605e5ed9..e4fdf0c0 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -58,6 +58,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 6f130f54..4b190ce3 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -50,6 +50,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index 324e1a77..886513c0 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -54,6 +54,7 @@ #include "glbl.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index aff76d0a..f8bb4aa6 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -46,6 +46,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 30b5834b..a37533ee 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -82,6 +82,7 @@ #include "omoracle.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /** */ DEF_OMOD_STATIC_DATA diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index ffdcc532..ab8e4d2c 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -49,6 +49,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 2687e7a3..56192579 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -45,6 +45,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 349e45aa..cf70381f 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -46,6 +46,7 @@ #include "debug.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omruleset/omruleset.c b/plugins/omruleset/omruleset.c index 0e0fc13b..c439bd83 100644 --- a/plugins/omruleset/omruleset.c +++ b/plugins/omruleset/omruleset.c @@ -49,6 +49,7 @@ #include "dirty.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* static data */ DEFobjCurrIf(ruleset); diff --git a/plugins/omsnmp/omsnmp.c b/plugins/omsnmp/omsnmp.c index b973b09d..443cfaab 100644 --- a/plugins/omsnmp/omsnmp.c +++ b/plugins/omsnmp/omsnmp.c @@ -47,6 +47,7 @@ #include "errmsg.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 929de703..dc9912e3 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -44,6 +44,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omtemplate/omtemplate.c b/plugins/omtemplate/omtemplate.c index 5577f8c6..1472ebeb 100644 --- a/plugins/omtemplate/omtemplate.c +++ b/plugins/omtemplate/omtemplate.c @@ -45,6 +45,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omtesting/omtesting.c b/plugins/omtesting/omtesting.c index c474bb41..6d178467 100644 --- a/plugins/omtesting/omtesting.c +++ b/plugins/omtesting/omtesting.c @@ -57,6 +57,7 @@ #include "cfsysline.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omudpspoof/omudpspoof.c b/plugins/omudpspoof/omudpspoof.c index 3ead5447..48d7a68e 100644 --- a/plugins/omudpspoof/omudpspoof.c +++ b/plugins/omudpspoof/omudpspoof.c @@ -82,6 +82,7 @@ MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/omuxsock/omuxsock.c b/plugins/omuxsock/omuxsock.c index c66e63aa..0e336c51 100644 --- a/plugins/omuxsock/omuxsock.c +++ b/plugins/omuxsock/omuxsock.c @@ -52,6 +52,7 @@ #include "unicode-helper.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ diff --git a/plugins/pmlastmsg/pmlastmsg.c b/plugins/pmlastmsg/pmlastmsg.c index 275f1c1f..259c5d41 100644 --- a/plugins/pmlastmsg/pmlastmsg.c +++ b/plugins/pmlastmsg/pmlastmsg.c @@ -47,6 +47,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.lastline") /* internal structures diff --git a/plugins/pmrfc3164sd/pmrfc3164sd.c b/plugins/pmrfc3164sd/pmrfc3164sd.c index 5598c025..53204ece 100644 --- a/plugins/pmrfc3164sd/pmrfc3164sd.c +++ b/plugins/pmrfc3164sd/pmrfc3164sd.c @@ -45,6 +45,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("contrib.rfc3164sd") /* internal structures -- cgit From 7bccae395b83416be6bea2de69fdfcce46995acf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 23 Feb 2011 16:22:17 +0000 Subject: fixed compile problems on Solaris --- plugins/imuxsock/imuxsock.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'plugins') diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index ff38852c..7ee413e7 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -606,7 +606,9 @@ static rsRetVal readSocket(lstn_t *pLstn) int iMaxLine; struct msghdr msgh; struct iovec msgiov; +# if HAVE_SCM_CREDENTIALS struct cmsghdr *cm; +# endif struct ucred *cred; uchar bufRcv[4096+1]; char aux[128]; @@ -630,11 +632,13 @@ static rsRetVal readSocket(lstn_t *pLstn) memset(&msgh, 0, sizeof(msgh)); memset(&msgiov, 0, sizeof(msgiov)); +# if HAVE_SCM_CREDENTIALS if(pLstn->bUseCreds) { memset(&aux, 0, sizeof(aux)); msgh.msg_control = aux; msgh.msg_controllen = sizeof(aux); } +# endif msgiov.iov_base = pRcv; msgiov.iov_len = iMaxLine; msgh.msg_iov = &msgiov; -- cgit From 68124ef1f63daef07ecf45d479ca9697cb099441 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 6 Mar 2011 15:18:01 +0100 Subject: fixed small regression from gnutls fd leak fix --- plugins/impstats/impstats.c | 1 + plugins/pmcisconames/pmcisconames.c | 1 + plugins/pmsnare/pmsnare.c | 1 + 3 files changed, 3 insertions(+) (limited to 'plugins') diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index 1312a4e8..aa98ae9d 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -49,6 +49,7 @@ #include "prop.h" MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP /* defines */ #define DEFAULT_STATS_PERIOD (5 * 60) diff --git a/plugins/pmcisconames/pmcisconames.c b/plugins/pmcisconames/pmcisconames.c index 4171e688..61688cbf 100644 --- a/plugins/pmcisconames/pmcisconames.c +++ b/plugins/pmcisconames/pmcisconames.c @@ -41,6 +41,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.cisconames") /* internal structures diff --git a/plugins/pmsnare/pmsnare.c b/plugins/pmsnare/pmsnare.c index 4a9880d4..f3658d11 100644 --- a/plugins/pmsnare/pmsnare.c +++ b/plugins/pmsnare/pmsnare.c @@ -58,6 +58,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.snare") /* internal structures -- cgit From 4d55e2481f51b59fff9a32f9392bb3e74da9789a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 6 Mar 2011 15:59:06 +0100 Subject: forgot to backport imdiag mem leak fix -- now done --- plugins/imdiag/imdiag.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'plugins') diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index b7a2a070..0a69ee43 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -292,6 +292,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) { int iMsgQueueSize; uchar *pszMsg; + uchar *pToFree = NULL; uchar cmdBuf[1024]; DEFiRet; @@ -303,6 +304,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) * before proceeding. */ CHKmalloc(pszMsg = MALLOC(sizeof(uchar) * (iLenMsg + 1))); + pToFree = pszMsg; memcpy(pszMsg, pRcv, iLenMsg); pszMsg[iLenMsg] = '\0'; @@ -322,6 +324,8 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) } finalize_it: + if(pToFree != NULL) + free(pToFree); RETiRet; } -- cgit From 53afa44de8d7796df46a12d66fd9866165779d94 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 9 Mar 2011 10:31:47 +0100 Subject: bugfix: (regression) omhdfs did no longer compile --- plugins/omhdfs/omhdfs.c | 1 + 1 file changed, 1 insertion(+) (limited to 'plugins') diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 9705b7fd..8b72747f 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -50,6 +50,7 @@ #include "hashtable_itr.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ -- cgit From 7fa1956da7ea3fa08d3187d3a5d883cfe5a67845 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Thu, 10 Mar 2011 03:23:58 +0100 Subject: bugfix: pmaixforwarded from fails to build Signed-off-by: Rainer Gerhards --- plugins/pmaixforwardedfrom/pmaixforwardedfrom.c | 1 + 1 file changed, 1 insertion(+) (limited to 'plugins') diff --git a/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c index 11634199..fa4a9087 100644 --- a/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c +++ b/plugins/pmaixforwardedfrom/pmaixforwardedfrom.c @@ -41,6 +41,7 @@ #include "unicode-helper.h" MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.aixforwardedfrom") /* internal structures -- cgit From 9cc963926bad3de9a78df51a19456f419c34492a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Mar 2011 14:46:47 +0100 Subject: bugfix: minor memory leak in omlibdbi (< 1k per instance and run) also testbench improvement (omlibdbi now also receives a couple of tests if we have a life MySQL server). --- plugins/omlibdbi/omlibdbi.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index f49bef42..7fcf9631 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -108,6 +108,11 @@ static void closeConn(instanceData *pData) BEGINfreeInstance CODESTARTfreeInstance closeConn(pData); + free(pData->drvrName); + free(pData->host); + free(pData->usrName); + free(pData->pwd); + free(pData->dbName); ENDfreeInstance @@ -171,7 +176,8 @@ static rsRetVal initConn(instanceData *pData, int bSilent) errmsg.LogError(0, RS_RET_SUSPENDED, "libdbi error: libdbi or libdbi drivers not present on this system - suspending."); ABORT_FINALIZE(RS_RET_SUSPENDED); } else if(iDrvrsLoaded < 0) { - errmsg.LogError(0, RS_RET_SUSPENDED, "libdbi error: libdbi could not be initialized - suspending."); + errmsg.LogError(0, RS_RET_SUSPENDED, "libdbi error: libdbi could not be " + "initialized (do you have any dbi drivers installed?) - suspending."); ABORT_FINALIZE(RS_RET_SUSPENDED); } bDbiInitialized = 1; /* we are done for the rest of our existence... */ @@ -367,6 +373,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr( (uchar *)"actionlibdbipassword", 0, eCmdHdlrGetWord, NULL, &pwd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr( (uchar *)"actionlibdbidbname", 0, eCmdHdlrGetWord, NULL, &dbName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr( (uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + DBGPRINTF("omlibdbi compiled with version %s loaded, libdbi version %s\n", VERSION, dbi_version()); ENDmodInit /* vim:set ai: -- cgit From 4671cad51221f1cb03822481e46669ffba3c6d95 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Sun, 13 Mar 2011 12:09:06 +0100 Subject: Fall back to epoll_create() if epoll_create1() is not available epoll_create1() was introduced in Linux kernel 2.6.27. If rsyslog was compiled on a newer kernel but run on a kernel older than 2.6.27, remote syslog fails. Apply a runtime check for epoll_create1() and fall back to epoll_create() in this case. Bug-Debian: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=617996 --- plugins/imptcp/imptcp.c | 13 ++++++++----- plugins/imudp/imudp.c | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) (limited to 'plugins') diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 3197564e..c4fbab3d 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -1047,17 +1047,20 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } -# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) - DBGPRINTF("imptcp uses epoll_create1()\n"); - epollfd = epoll_create1(EPOLL_CLOEXEC); -# else +#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imptcp uses epoll_create1()\n"); + epollfd = epoll_create1(EPOLL_CLOEXEC); + if(epollfd < 0 && errno ENOSYS) +#endif + { DBGPRINTF("imptcp uses epoll_create()\n"); /* reading the docs, the number of epoll events passed to * epoll_create() seems not to be used at all in kernels. So * we just provide "a" number, happens to be 10. */ epollfd = epoll_create(10); -# endif + } + if(epollfd < 0) { errmsg.LogError(0, RS_RET_EPOLL_CR_FAILED, "error: epoll_create() failed"); ABORT_FINALIZE(RS_RET_NO_RUN); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 56cdab28..a5002591 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -453,13 +453,16 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) CHKmalloc(udpEPollEvt = calloc(udpLstnSocks[0], sizeof(struct epoll_event))); -# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) - DBGPRINTF("imudp uses epoll_create1()\n"); - efd = epoll_create1(EPOLL_CLOEXEC); -# else +#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imudp uses epoll_create1()\n"); + efd = epoll_create1(EPOLL_CLOEXEC); + if(efd < 0 && errno == ENOSYS) +#endif + { DBGPRINTF("imudp uses epoll_create()\n"); efd = epoll_create(NUM_EPOLL_EVENTS); -# endif + } + if(efd < 0) { DBGPRINTF("epoll_create1() could not create fd\n"); ABORT_FINALIZE(RS_RET_IO_ERROR); -- cgit From 91d4e54a6707fa87298e367fb734fed931310b40 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 14 Mar 2011 09:24:57 +0100 Subject: fixed compile problem from epoll_create patch --- plugins/imptcp/imptcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index c4fbab3d..e8cffbb0 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -1050,7 +1050,7 @@ CODESTARTwillRun #if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) DBGPRINTF("imptcp uses epoll_create1()\n"); epollfd = epoll_create1(EPOLL_CLOEXEC); - if(epollfd < 0 && errno ENOSYS) + if(epollfd < 0 && errno == ENOSYS) #endif { DBGPRINTF("imptcp uses epoll_create()\n"); -- cgit From 32db6291bc310062cffd3266fc297c7b13c571a7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 16 Mar 2011 16:59:49 +0100 Subject: backporting patch for clean make distcheck --- plugins/imdiag/imdiag.c | 3 +++ plugins/imfile/imfile.c | 8 ++++++++ 2 files changed, 11 insertions(+) (limited to 'plugins') diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 0a69ee43..404cebc7 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -433,6 +433,9 @@ CODESTARTmodExit net.DestructPermittedPeers(&pPermPeersRoot); } + /* free some globals to keep valgrind happy */ + free(pszInputName); + /* release objects we used */ objRelease(net, LM_NET_FILENAME); objRelease(netstrm, LM_NETSTRMS_FILENAME); diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index acb58dad..cac3a55d 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -331,6 +331,11 @@ ENDrunInput */ BEGINwillRun CODESTARTwillRun + /* free config variables we do no longer needed */ + free(pszFileName); + free(pszFileTag); + free(pszStateFile); + if(iFilPtr == 0) { errmsg.LogError(0, RS_RET_NO_RUN, "No files configured to be monitored"); ABORT_FINALIZE(RS_RET_NO_RUN); @@ -398,6 +403,9 @@ CODESTARTafterRun persistStrmState(&files[i]); strm.Destruct(&(files[i].pStrm)); } + free(files[i].pszFileName); + free(files[i].pszTag); + free(files[i].pszStateFile); } if(pInputName != NULL) -- cgit From 68bde6252d22cc4e88edc2b51523726bb858c818 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Mar 2011 13:19:38 +0100 Subject: prepared skeleton for new strgen module --- plugins/sm_cust_bindcdr/Makefile.am | 6 ++ plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 138 ++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 plugins/sm_cust_bindcdr/Makefile.am create mode 100644 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c (limited to 'plugins') diff --git a/plugins/sm_cust_bindcdr/Makefile.am b/plugins/sm_cust_bindcdr/Makefile.am new file mode 100644 index 00000000..1f71d499 --- /dev/null +++ b/plugins/sm_cust_bindcdr/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = sm_cust_bindcdr.la + +sm_cust_bindcdr_la_SOURCES = sm_cust_bindcdr.c +sm_cust_bindcdr_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +sm_cust_bindcdr_la_LDFLAGS = -module -avoid-version +sm_cust_bindcdr_la_LIBADD = diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c new file mode 100644 index 00000000..cf3cdeef --- /dev/null +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -0,0 +1,138 @@ +/* sm_cust_bindcdr.c + * This is a custom developed plugin to process bind information into + * a specific SQL statement. While the actual processing may be too specific + * to be of general use, this module serves as a template on how this type + * of processing can be done. + * + * Format generated: + * "%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" + * Note that this is the same as smtradfile.c, except that we do have a RFC3339 timestamp. However, + * we have copied over the code from there, it is too simple to go through all the hassle + * of having a single code base. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2011-03-17 by RGerhards + * + * Copyright 2011 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" + +MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP +STRGEN_NAME("Custom_BindCDR") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA + + +/* config data */ + + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + */ +BEGINstrgen + register int iBuf; + uchar *pTimeStamp; + size_t lenTimeStamp; + uchar *pHOSTNAME; + size_t lenHOSTNAME; + uchar *pTAG; + int lenTAG; + uchar *pMSG; + size_t lenMSG; + size_t lenTotal; +CODESTARTstrgen + /* first obtain all strings and their length (if not fixed) */ + pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); + lenTimeStamp = ustrlen(pTimeStamp); + pHOSTNAME = (uchar*) getHOSTNAME(pMsg); + lenHOSTNAME = getHOSTNAMELen(pMsg); + getTAG(pMsg, &pTAG, &lenTAG); + pMSG = getMSG(pMsg); + lenMSG = getMSGLen(pMsg); + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2; + if(pMSG[0] != ' ') + ++lenTotal; /* then we need to introduce one additional space */ + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + memcpy(*ppBuf, pTimeStamp, lenTimeStamp); + iBuf = lenTimeStamp; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME); + iBuf += lenHOSTNAME; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pTAG, lenTAG); + iBuf += lenTAG; + + if(pMSG[0] != ' ') + *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf + iBuf, pMSG, lenMSG); + iBuf += lenMSG; + + /* trailer */ + *(*ppBuf + iBuf++) = '\n'; + *(*ppBuf + iBuf) = '\0'; + +finalize_it: +ENDstrgen + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(sm_cust_bindcdr) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + + dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION); +ENDmodInit -- cgit From 459f0d094fcb48fd58204ad3527f931715839bcb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Mar 2011 14:59:18 +0100 Subject: setup test environment and test code in order to look at... bug http://bugzilla.adiscon.com/show_bug.cgi?id=195 That bug currently prevents strgen's to be used with databases --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 45 ++++++++++--------------------- 1 file changed, 14 insertions(+), 31 deletions(-) (limited to 'plugins') diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index cf3cdeef..5ea7e653 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -63,56 +63,39 @@ DEF_SMOD_STATIC_DATA * needed (including their length) and then calculating the actual space required. So when we * finally copy, we know exactly what we need. So we do at most one alloc. */ +//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" +#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('" BEGINstrgen register int iBuf; uchar *pTimeStamp; size_t lenTimeStamp; - uchar *pHOSTNAME; - size_t lenHOSTNAME; - uchar *pTAG; - int lenTAG; - uchar *pMSG; - size_t lenMSG; size_t lenTotal; CODESTARTstrgen /* first obtain all strings and their length (if not fixed) */ pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); lenTimeStamp = ustrlen(pTimeStamp); - pHOSTNAME = (uchar*) getHOSTNAME(pMsg); - lenHOSTNAME = getHOSTNAMELen(pMsg); - getTAG(pMsg, &pTAG, &lenTAG); - pMSG = getMSG(pMsg); - lenMSG = getMSGLen(pMsg); /* calculate len, constants for spaces and similar fixed strings */ - lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2; - if(pMSG[0] != ' ') - ++lenTotal; /* then we need to introduce one additional space */ + lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2; /* now make sure buffer is large enough */ if(lenTotal >= *pLenBuf) CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); /* and concatenate the resulting string */ - memcpy(*ppBuf, pTimeStamp, lenTimeStamp); - iBuf = lenTimeStamp; - *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1); + iBuf = sizeof(SQL_STMT) - 1; - memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME); - iBuf += lenHOSTNAME; - *(*ppBuf + iBuf++) = ' '; + // SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); - memcpy(*ppBuf + iBuf, pTAG, lenTAG); - iBuf += lenTAG; + memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp); + iBuf += lenTimeStamp; + memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1); + iBuf += sizeof("', '") - 1; - if(pMSG[0] != ' ') - *(*ppBuf + iBuf++) = ' '; - memcpy(*ppBuf + iBuf, pMSG, lenMSG); - iBuf += lenMSG; - - /* trailer */ - *(*ppBuf + iBuf++) = '\n'; - *(*ppBuf + iBuf) = '\0'; + /* end of SQL statement/trailer (NUL is contained in string!) */ + memcpy(*ppBuf + iBuf, "');", sizeof("');")); + iBuf += sizeof("');"); finalize_it: ENDstrgen @@ -129,7 +112,7 @@ CODEqueryEtryPt_STD_SMOD_QUERIES ENDqueryEtryPt -BEGINmodInit(sm_cust_bindcdr) +BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr -- cgit From 41d79b8fead00f85614547606b0c1117f3efa04e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 21 Mar 2011 10:34:43 +0100 Subject: bugfix: strgen could not be used together with database outputs because the sql/stdsql option could not be specified. This has been solved by permitting the strgen to include the opton inside its name. closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195 --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index 5ea7e653..d1634252 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -49,7 +49,7 @@ MODULE_TYPE_STRGEN MODULE_TYPE_NOKEEP -STRGEN_NAME("Custom_BindCDR") +STRGEN_NAME("Custom_BindCDR,sql") /* internal structures */ -- cgit From 5808af91e7e5e552a5cde4603687685978934626 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Mar 2011 12:17:20 +0100 Subject: added internal processing of BIND records still some questions outstanding, so actual SQL statement is not usable at the moment --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 121 ++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 6 deletions(-) (limited to 'plugins') diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index d1634252..be01db77 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -37,6 +37,7 @@ #include "config.h" #include "rsyslog.h" #include +#include #include #include #include @@ -64,19 +65,112 @@ DEF_SMOD_STATIC_DATA * finally copy, we know exactly what we need. So we do at most one alloc. */ //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" -#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('" +//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('" +#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('" +#define ADD_SQL_DELIM \ + memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \ + iBuf += sizeof("', '") - 1; +#define SQL_STMT_END "');\n" BEGINstrgen register int iBuf; + uchar *psz; uchar *pTimeStamp; + uchar szClient[64]; + unsigned lenClient; + uchar szView[64]; + unsigned lenView; + uchar szQuery[64]; + unsigned lenQuery; + uchar szIP[64]; + unsigned lenIP; size_t lenTimeStamp; size_t lenTotal; CODESTARTstrgen + /* first create an empty statement. This is to be replaced if + * we have better data to fill in. + */ + /* now make sure buffer is large enough */ + if(*pLenBuf < 2) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, 2)); + memcpy(*ppBuf, ";", sizeof(";")); + /* first obtain all strings and their length (if not fixed) */ pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); lenTimeStamp = ustrlen(pTimeStamp); + + /* "client" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: client part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("client ") - 1; /* skip "label" */ + for( lenClient = 0 + ; *psz && *psz != '#' && lenClient < sizeof(szClient) - 1 + ; ++lenClient) { + szClient[lenClient] = *psz++; + } + szClient[lenClient] = '\0'; + } + + /* "view" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "view "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: view part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("view ") - 1; /* skip "label" */ + for( lenView = 0 + ; *psz && *psz != ':' && lenView < sizeof(szView) - 1 + ; ++lenView) { + szView[lenView] = *psz++; + } + szView[lenView] = '\0'; + } + + /* "query" - we must extract just the number, and in reverse! */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "query: "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: query part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("query: ") - 1; /* skip "label" */ + /* first find end-of-string to process */ + while(*psz && (isdigit(*psz) || *psz == '.')) { +dbgprintf("XXXX: step 1: %c\n", *psz); + psz++; + } + /* now shuffle data */ + for( lenQuery = 0 + ; *psz && *psz != ' ' && lenQuery < sizeof(szQuery) - 1 + ; --psz) { + if(isdigit(*psz)) + szQuery[lenQuery++] = *psz; + } + szQuery[lenQuery] = '\0'; + } + + /* "ip" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + ("); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: ip part in msg missing\n"); + FINALIZE; + } else { + psz += sizeof("IN TXT + (") - 1; /* skip "label" */ + for( lenIP = 0 + ; *psz && *psz != ')' && lenIP < sizeof(szIP) - 1 + ; ++lenIP) { + szIP[lenIP] = *psz++; + } + szIP[lenIP] = '\0'; + } + + + /* --- strings extracted ---- */ /* calculate len, constants for spaces and similar fixed strings */ - lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2; + lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5 + + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; /* now make sure buffer is large enough */ if(lenTotal >= *pLenBuf) @@ -90,12 +184,27 @@ CODESTARTstrgen memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp); iBuf += lenTimeStamp; - memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1); - iBuf += sizeof("', '") - 1; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szClient, lenClient); + iBuf += lenClient; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szView, lenView); + iBuf += lenView; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szQuery, lenQuery); + iBuf += lenQuery; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szIP, lenIP); + iBuf += lenIP; + ADD_SQL_DELIM /* end of SQL statement/trailer (NUL is contained in string!) */ - memcpy(*ppBuf + iBuf, "');", sizeof("');")); - iBuf += sizeof("');"); + memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END)); + iBuf += sizeof(SQL_STMT_END); finalize_it: ENDstrgen -- cgit From 0d80f4e4612fb209cc9dda5faf7b71cd261c0c20 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Mar 2011 14:22:38 +0100 Subject: sm_bind_cdr: added capability to configure "allowed IPs" --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 83 +++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index be01db77..005a4ba9 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -43,10 +43,12 @@ #include #include "conf.h" #include "syslogd-types.h" +#include "cfsysline.h" #include "template.h" #include "msg.h" #include "module-template.h" #include "unicode-helper.h" +#include "errmsg.h" MODULE_TYPE_STRGEN MODULE_TYPE_NOKEEP @@ -55,10 +57,65 @@ STRGEN_NAME("Custom_BindCDR,sql") /* internal structures */ DEF_SMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +/* list of "allowed" IPs */ +typedef struct allowedip_s { + uchar *pszIP; + struct allowedip_s *next; +} allowedip_t; + +static allowedip_t *root; /* config data */ +/* check if the provided IP is (already) in the allowed list + */ +static int +isAllowed(uchar *pszIP) +{ + allowedip_t *pallow; + int ret = 0; + + for(pallow = root ; pallow != NULL ; pallow = pallow->next) { + DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP); + if(!ustrcmp(pallow->pszIP, pszIP)) { + ret = 1; + goto finalize_it; + } + } +finalize_it: return ret; +} + +/* This function is called to add an additional allowed IP. It adds + * the IP to the linked list of them. An error is emitted if the IP + * already exists. + */ +static rsRetVal addAllowedIP(void __attribute__((unused)) *pVal, uchar *pNewVal) +{ + allowedip_t *pNew; + DEFiRet; + + if(isAllowed(pNewVal)) { + errmsg.LogError(0, NO_ERRCODE, "error: allowed IP '%s' already configured " + "duplicate ignored", pNewVal); + ABORT_FINALIZE(RS_RET_ERR); + } + + CHKmalloc(pNew = malloc(sizeof(allowedip_t))); + pNew->pszIP = pNewVal; + pNew->next = root; + root = pNew; + DBGPRINTF("sm_cust_bindcdr: allowed IP '%s' added.\n", pNewVal); + +finalize_it: + if(iRet != RS_RET_OK) { + free(pNewVal); + } + + RETiRet; +} /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings * needed (including their length) and then calculating the actual space required. So when we @@ -72,9 +129,10 @@ DEF_SMOD_STATIC_DATA iBuf += sizeof("', '") - 1; #define SQL_STMT_END "');\n" BEGINstrgen - register int iBuf; + int iBuf; uchar *psz; uchar *pTimeStamp; + size_t lenTimeStamp; uchar szClient[64]; unsigned lenClient; uchar szView[64]; @@ -83,7 +141,6 @@ BEGINstrgen unsigned lenQuery; uchar szIP[64]; unsigned lenIP; - size_t lenTimeStamp; size_t lenTotal; CODESTARTstrgen /* first create an empty statement. This is to be replaced if @@ -137,7 +194,6 @@ CODESTARTstrgen psz += sizeof("query: ") - 1; /* skip "label" */ /* first find end-of-string to process */ while(*psz && (isdigit(*psz) || *psz == '.')) { -dbgprintf("XXXX: step 1: %c\n", *psz); psz++; } /* now shuffle data */ @@ -168,6 +224,14 @@ dbgprintf("XXXX: step 1: %c\n", *psz); /* --- strings extracted ---- */ + /* now check if the IP is "allowed", in which case we should not + * insert into the database. + */ + if(isAllowed(szIP)) { + DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n"); + FINALIZE; + } + /* calculate len, constants for spaces and similar fixed strings */ lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; @@ -211,7 +275,16 @@ ENDstrgen BEGINmodExit + allowedip_t *pallow, *pdel; CODESTARTmodExit + for(pallow = root ; pallow != NULL ; ) { + pdel = pallow; + pallow = pallow->next; + free(pdel->pszIP); + free(pdel); + } + + objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -225,6 +298,10 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + root = NULL; + CHKiRet(omsdRegCFSLineHdlr((uchar *)"sgcustombindcdrallowedip", 0, eCmdHdlrGetWord, + addAllowedIP, NULL, STD_LOADABLE_MODULE_ID)); dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION); ENDmodInit -- cgit From 0fcbbfbe3b40e2739531cfa3a308c6fc8681ee7f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 28 Mar 2011 16:09:07 +0200 Subject: sm_cust_bindcdr: custom date parsing added --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 100 +++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 10 deletions(-) (limited to 'plugins') diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index 005a4ba9..fa16acb5 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -120,6 +120,8 @@ finalize_it: /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings * needed (including their length) and then calculating the actual space required. So when we * finally copy, we know exactly what we need. So we do at most one alloc. + * An actual message sample for what we intend to parse is (one line): + <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96) */ //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" //#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('" @@ -131,8 +133,12 @@ finalize_it: BEGINstrgen int iBuf; uchar *psz; - uchar *pTimeStamp; - size_t lenTimeStamp; + uchar szDate[64]; + unsigned lenDate; + uchar szTime[64]; + unsigned lenTime; + uchar szMSec[64]; + unsigned lenMSec; uchar szClient[64]; unsigned lenClient; uchar szView[64]; @@ -152,9 +158,76 @@ CODESTARTstrgen memcpy(*ppBuf, ";", sizeof(";")); /* first obtain all strings and their length (if not fixed) */ - pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); - lenTimeStamp = ustrlen(pTimeStamp); - + /* Note that there are two date fields present, one in the header + * and one more in the actual message. We use the one from the message + * and parse that our. We check validity based on some fixe fields. In- + * depth verification is probably not worth the effort (CPU time), because + * we do various other checks on the message format below). + */ + psz = getMSG(pMsg); + if(psz[0] == ' ' && psz[3] == '-' && psz[7] == '-') { + memcpy(szDate, psz+8, 4); + szDate[4] = '-'; + if(!strncmp((char*)psz+4, "Jan", 3)) { + szDate[5] = '0'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Feb", 3)) { + szDate[5] = '0'; + szDate[6] = '2'; + } else if(!strncmp((char*)psz+4, "Mar", 3)) { + szDate[5] = '0'; + szDate[6] = '3'; + } else if(!strncmp((char*)psz+4, "Apr", 3)) { + szDate[5] = '0'; + szDate[6] = '4'; + } else if(!strncmp((char*)psz+4, "May", 3)) { + szDate[5] = '0'; + szDate[6] = '5'; + } else if(!strncmp((char*)psz+4, "Jun", 3)) { + szDate[5] = '0'; + szDate[6] = '6'; + } else if(!strncmp((char*)psz+4, "Jul", 3)) { + szDate[5] = '0'; + szDate[6] = '7'; + } else if(!strncmp((char*)psz+4, "Aug", 3)) { + szDate[5] = '0'; + szDate[6] = '8'; + } else if(!strncmp((char*)psz+4, "Sep", 3)) { + szDate[5] = '0'; + szDate[6] = '9'; + } else if(!strncmp((char*)psz+4, "Oct", 3)) { + szDate[5] = '1'; + szDate[6] = '0'; + } else if(!strncmp((char*)psz+4, "Nov", 3)) { + szDate[5] = '1'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Dec", 3)) { + szDate[5] = '1'; + szDate[6] = '2'; + } + szDate[7] = '-'; + szDate[8] = psz[1]; + szDate[9] = psz[2]; + szDate[10] = '\0'; + lenDate = 10; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + FINALIZE; + } + + /* now time (pull both regular time and ms) */ + if(psz[12] == ' ' && psz[15] == ':' && psz[18] == ':' && psz[21] == '.' && psz[25] == ' ') { + memcpy(szTime, (char*)psz+13, 8); + szTime[9] = '\0'; + lenTime = 8; + memcpy(szMSec, (char*)psz+22, 3); + szMSec[4] = '\0'; + lenMSec = 3; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + FINALIZE; + } + /* "client" */ psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); if(psz == NULL) { @@ -233,8 +306,8 @@ CODESTARTstrgen } /* calculate len, constants for spaces and similar fixed strings */ - lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5 - + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; + lenTotal = lenDate + lenTime + lenMSec + lenClient + lenView + lenQuery + + lenIP + 7 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; /* now make sure buffer is large enough */ if(lenTotal >= *pLenBuf) @@ -246,8 +319,16 @@ CODESTARTstrgen // SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); - memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp); - iBuf += lenTimeStamp; + memcpy(*ppBuf + iBuf, szDate, lenDate); + iBuf += lenDate; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szTime, lenTime); + iBuf += lenTime; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szMSec, lenMSec); + iBuf += lenMSec; ADD_SQL_DELIM memcpy(*ppBuf + iBuf, szClient, lenClient); @@ -264,7 +345,6 @@ CODESTARTstrgen memcpy(*ppBuf + iBuf, szIP, lenIP); iBuf += lenIP; - ADD_SQL_DELIM /* end of SQL statement/trailer (NUL is contained in string!) */ memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END)); -- cgit From 030fd145a3d8759367b73cc66e03023ff131c951 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 29 Mar 2011 11:50:32 +0200 Subject: did mapping of data items to database table columns to facilitate testing --- plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) (limited to 'plugins') diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c index fa16acb5..3fe96ac4 100644 --- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -79,7 +79,6 @@ isAllowed(uchar *pszIP) int ret = 0; for(pallow = root ; pallow != NULL ; pallow = pallow->next) { - DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP); if(!ustrcmp(pallow->pszIP, pszIP)) { ret = 1; goto finalize_it; @@ -123,9 +122,7 @@ finalize_it: * An actual message sample for what we intend to parse is (one line): <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96) */ -//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('" -//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('" -#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('" +#define SQL_STMT "INSERT INTO CDR(`Date`,`Time`, timeMS, client, view, query, ip) VALUES ('" #define ADD_SQL_DELIM \ memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \ iBuf += sizeof("', '") - 1; @@ -212,7 +209,7 @@ CODESTARTstrgen lenDate = 10; } else { dbgprintf("Custom_BindCDR: date part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } /* now time (pull both regular time and ms) */ @@ -225,14 +222,14 @@ CODESTARTstrgen lenMSec = 3; } else { dbgprintf("Custom_BindCDR: date part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } /* "client" */ psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); if(psz == NULL) { dbgprintf("Custom_BindCDR: client part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("client ") - 1; /* skip "label" */ for( lenClient = 0 @@ -247,7 +244,7 @@ CODESTARTstrgen psz = (uchar*) strstr((char*) getMSG(pMsg), "view "); if(psz == NULL) { dbgprintf("Custom_BindCDR: view part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("view ") - 1; /* skip "label" */ for( lenView = 0 @@ -262,10 +259,10 @@ CODESTARTstrgen psz = (uchar*) strstr((char*) getMSG(pMsg), "query: "); if(psz == NULL) { dbgprintf("Custom_BindCDR: query part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("query: ") - 1; /* skip "label" */ - /* first find end-of-string to process */ + /* first find end-of-strihttp://www.rsyslog.com/doc/omruleset.htmlng to process */ while(*psz && (isdigit(*psz) || *psz == '.')) { psz++; } @@ -283,7 +280,7 @@ CODESTARTstrgen psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + ("); if(psz == NULL) { dbgprintf("Custom_BindCDR: ip part in msg missing\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } else { psz += sizeof("IN TXT + (") - 1; /* skip "label" */ for( lenIP = 0 @@ -302,7 +299,7 @@ CODESTARTstrgen */ if(isAllowed(szIP)) { DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n"); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } /* calculate len, constants for spaces and similar fixed strings */ @@ -317,8 +314,6 @@ CODESTARTstrgen memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1); iBuf = sizeof(SQL_STMT) - 1; - // SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); - memcpy(*ppBuf + iBuf, szDate, lenDate); iBuf += lenDate; ADD_SQL_DELIM -- cgit From 9003632c7df5e6020b06b7d19e1c226ea46640de Mon Sep 17 00:00:00 2001 From: Christian Kastner Date: Tue, 29 Mar 2011 12:00:56 +0200 Subject: Force ANSI SQL treatment of strings for Postgres Signed-off-by: Rainer Gerhards --- plugins/ompgsql/ompgsql.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index ab8e4d2c..ea4b4b75 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -149,8 +149,13 @@ static rsRetVal initPgSQL(instanceData *pData, int bSilent) dbgprintf("host=%s dbname=%s uid=%s\n",pData->f_dbsrv,pData->f_dbname,pData->f_dbuid); + /* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may + * get all sorts of side effects (e.g.: backslash escapes) and warnings + */ + const char *PgConnectionOptions = "-c standard_conforming_strings=on"; + /* Connect to database */ - if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, NULL, NULL, + if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, PgConnectionOptions, NULL, pData->f_dbname, pData->f_dbuid, pData->f_dbpwd)) == NULL) { reportDBError(pData, bSilent); closePgSQL(pData); /* ignore any error we may get */ -- cgit From b59f8fd65d4f464ce3673439725db1606ec31329 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 30 Mar 2011 11:19:22 +0200 Subject: fixed problem in testbench & added new test (not yet integrated) The test is a setup scenario for this bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=241 --- plugins/imdiag/imdiag.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 404cebc7..ed6ef509 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -205,7 +205,7 @@ doInjectMsg(int iNum) DEFiRet; snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), - "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum); + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum); datetime.getCurrTime(&stTime, &ttGenTime); /* we now create our own message object and submit it to the queue */ @@ -247,6 +247,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); + DBGPRINTF("imdiag: %d messages injected\n", nMsgs); finalize_it: RETiRet; @@ -279,6 +280,7 @@ waitMainQEmpty(tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "mainqueue empty\n")); + DBGPRINTF("imdiag: mainqueue empty\n"); finalize_it: RETiRet; @@ -314,6 +316,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); + DBGPRINTF("imdiag: %d messages in main queue\n", iMsgQueueSize); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) { CHKiRet(waitMainQEmpty(pSess)); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) { -- cgit