diff options
60 files changed, 1749 insertions, 180 deletions
@@ -1,4 +1,44 @@ --------------------------------------------------------------------------- +Version 5.5.2 [DEVEL] (rgerhards), 2009-11-?? +- worked around an issue where omfile failed to compile on32 bit platforms + under some circumstances (this smells like a gcc problem, but a simple + solution was available). Thanks to Kenneth Marshall for some advice. +- extended testbench +- bugfix: hostname accidently set to IP address for some message sources, + for example imudp. Thanks to Anton for reporting this bug. [imported v4] +- bugfix: ompgsql had problems with transaction support, what actually + rendered it unsuable. Thanks to forum user "horhe" for alerting me + on this bug and helping to debug/fix it! [imported from 5.3.6] +- bugfix: $CreateDirs variable not properly initialized, default thus + was random (but most often "on") [imported from v3] +--------------------------------------------------------------------------- +Version 5.5.1 [DEVEL] (rgerhards), 2009-11-27 +- introduced the ablity for netstream drivers to utilize an epoll interface + This offers increased performance and removes the select() FDSET size + limit from imtcp. Note that we fall back to select() if there is no + epoll netstream drivers. So far, an epoll driver has only been + implemented for plain tcp syslog, the rest will follow once the code + proves well in practice AND there is demand. +- re-implemented $EscapeControlCharacterTab config directive + Based on Jonathan Bond-Caron's patch for v4. This now also includes some + automatted tests. +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +--------------------------------------------------------------------------- +Version 5.5.0 [DEVEL] (rgerhards), 2009-11-18 +- moved DNS resolution code out of imudp and into the backend processing + Most importantly, DNS resolution now never happens if the resolved name + is not required. Note that this applies to imudp - for the other inputs, + DNS resolution almost comes for free, so we do not do it there. However, + the new method has been implemented in a generic way and as such may + also be used by other modules in the future. +- added option to use unlimited-size select() calls + Thanks to varmjofekoj for the patch + This is not done in imudp, as it natively supports epoll(). +- doc: improved description of what loadable modules can do +--------------------------------------------------------------------------- Version 5.3.7 [BETA] (rgerhards), 2010-01-?? - bugfix: potential segfaults during queue shutdown (bugs require certain non-standard settings to appear) @@ -314,7 +354,17 @@ Version 4.7.0 [v4-devel] (rgerhards), 2009-09-?? - added new config directive $omfileForceChown to (try to) fix some broken system configs. See ticket for details: http://bugzilla.adiscon.com/show_bug.cgi?id=150 -- imported changes from 4.5.6 and below +- added $EscapeControlCharacterTab config directive + Thanks to Jonathan Bond-Caron for the patch. +- added option to use unlimited-size select() calls + Thanks to varmjofekoj for the patch +- debugondemand mode caused backgrounding to fail - close to a bug, but I'd + consider the ability to background in this mode a new feature... +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +- imported changes from 4.5.7 and below +- bugfix: potential segfault when -p command line option was used + Thanks for varmojfekoj for pointing me at this bug. --------------------------------------------------------------------------- Version 4.5.8 [v4-beta] (rgerhards), 2010-01-?? - bugfix: -d did not enable display of debug messages diff --git a/Makefile.am b/Makefile.am index 5f9d35fe..52a716fd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -99,6 +99,10 @@ if ENABLE_OMRULESET SUBDIRS += plugins/omruleset endif +if ENABLE_OMDBALERTING +SUBDIRS += plugins/omdbalerting +endif + if ENABLE_OMUDPSPOOF SUBDIRS += plugins/omudpspoof endif @@ -901,6 +901,7 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem, int *pbShutdownImmedi bDone = 0; do { localRet = tryDoAction(pAction, pBatch, &nElem, pbShutdownImmediate); +dbgprintf("submitBatch: state of tryDoAction %d\n", localRet); if(localRet == RS_RET_FORCE_TERM) FINALIZE; if( localRet == RS_RET_OK diff --git a/configure.ac b/configure.ac index 5186eb26..ea3fb1c8 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.3.6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.5.2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -347,6 +347,21 @@ AC_ARG_ENABLE([fsstnd], ]) +# support for unlimited select() syscall +AC_ARG_ENABLE(unlimited_select, + [AS_HELP_STRING([--enable-unlimited-select],[Enable unlimited select() syscall @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_unlimited_select="yes" ;; + no) enable_unlimited_select="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-unlimited-select) ;; + esac], + [enable_unlimited_select="no"] +) +if test "$enable_unlimited_select" = "yes"; then + AC_DEFINE(USE_UNLIMITED_SELECT, 1, [If defined, the select() syscall won't be limited to a particular number of file descriptors.]) +fi + + # debug AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], @@ -806,6 +821,20 @@ AC_ARG_ENABLE(omruleset, AM_CONDITIONAL(ENABLE_OMRULESET, test x$enable_omruleset = xyes) +# settings for omdbalerting +AC_ARG_ENABLE(omdbalerting, + [AS_HELP_STRING([--enable-omdbalerting],[Compiles omdbalerting module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omdbalerting="yes" ;; + no) enable_omdbalerting="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omdbalerting) ;; + esac], + [enable_omdbalerting=no] +) +AM_CONDITIONAL(ENABLE_OMDBALERTING, test x$enable_omdbalerting = xyes) + + + # building the GUI (mostly for diagnostic reasons) AC_ARG_ENABLE(gui, [AS_HELP_STRING([--enable-gui],[Enable GUI programs @<:@default=no@:>@])], @@ -897,6 +926,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/omruleset/Makefile \ + plugins/omdbalerting/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ @@ -926,6 +956,7 @@ echo " rsyslog runtime will be built: $enable_rsyslogrt" echo " rsyslogd will be built: $enable_rsyslogd" echo " GUI components will be built: $enable_gui" echo " custom module 1 will be built: $enable_cust1" +echo " Unlimited select() support enabled: $enable_unlimited_select" echo echo "---{ input plugins }---" echo " Klog functionality enabled: $enable_klog ($os_type)" @@ -938,6 +969,7 @@ echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" echo " omruleset module will be compiled: $enable_omruleset" +echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" echo " output template module will be compiled: $enable_omtemplate" echo diff --git a/doc/Makefile.am b/doc/Makefile.am index 661e9c57..2d451102 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -122,6 +122,7 @@ html_files = \ grfx_files = \ rsyslog_confgraph_complex.png\ rsyslog_confgraph_std.png \ + module_workflow.png \ direct_queue0.png \ direct_queue1.png \ direct_queue2.png \ diff --git a/doc/build_from_repo.html b/doc/build_from_repo.html index 8d3b20fe..a06863e1 100644 --- a/doc/build_from_repo.html +++ b/doc/build_from_repo.html @@ -43,12 +43,37 @@ you downloaded an official distribution tarball (see the <a href="install.html">rsyslog install guide</a>, starting at step 2, for further details about that). +<h2>Special Compile-Time Options</h2> +<p>On some platforms, compile-time issues occur, like the one shown below: +<p><pre><code> +make[2]: Entering directory `/home/az/RSyslog/rsyslog-5.5.0/tools' + CCLD rsyslogd +rsyslogd-omfile.o: In function `getClockFileAccess': +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +</code></pre> +<p>Note that the exact error messages can be different. These type of errors stem down to +atomic instruction support in GCC, which is somewhat depending on the machine architecture it +compiles code for. Very old machines (like the original i386) do not even at all provide support +for these instructions. +<p>The availability of atomic instructions is vital for rsyslog - it can not be built without them. +Consequently, there is a configure check included for them. But under some circumstances, +GCC seems to report they are available, but does not provide implementations for +all of them (at least this is my observation...). The simple cure is to make sure that +GCC generates code for a modern-enough architecture. This, for example, can be done as +follows: +<p><pre><code> +./configure CFLAGS="-march=i586 -mcpu=i686" --enable-imfile ... (whatever you need) +</code></pre> +<p>These settings should resolve the issue . + <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 1.2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/manual.html b/doc/manual.html index e2b171d8..c672af58 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 5.3.6 (beta branch) of rsyslog.</b> +<p><b>This documentation is for version 5.5.1 (devel branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might @@ -69,6 +69,7 @@ syslog sender over NAT</a> (online only)</li> <li><a href="debug.html">debug support in rsyslog</a></li> <li>Developer Documentation <ul> + <li><a href="build_from_repo.html">building rsyslog from the source repository</a></li> <li><a href="dev_oplugins.html">writing rsyslog output plugins</a></li> <li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> </ul></li> diff --git a/doc/module_workflow.png b/doc/module_workflow.png Binary files differnew file mode 100644 index 00000000..e1a72e96 --- /dev/null +++ b/doc/module_workflow.png diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index beb90e02..8796de65 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -141,6 +141,7 @@ our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a <li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li> <li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li> <li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> +<li><b>$EscapeControlCharactersOnReceive</b> [<b>on</b>|off] - escape USASCII HT character</li> <li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> <li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li> <li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li> diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 4ce62b38..a246d0ca 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -19,8 +19,16 @@ modules solve your need, you may consider writing one or have one written for you by <a href="http://www.rsyslog.com/professional-services">Adiscon's professional services for rsyslog</a> </b>(this often is a very cost-effective and efficient way of getting what you need). +<p>There exist different classes of loadable modules: +<ul> +<li><a href="rsyslog_conf_modules.html#im">Input Modules</a> +<li><a href="rsyslog_conf_modules.html#om">Output Modules</a> +<li><a href="rsyslog_conf_modules.html#pm">Parser Modules</a> +<li><a href="rsyslog_conf_modules.html#mm">Message Modification Modules</a> +<li><a href="rsyslog_conf_modules.html#lm">Library Modules</a> +</ul> -<h2>Input Modules</h2> +<a name"im"></a><h2>Input Modules</h2> <p>Input modules are used to gather messages from various sources. They interface to message generators. <ul> @@ -35,7 +43,7 @@ to message generators. <li><a href="im3195.html">im3195</a> - accepts syslog messages via RFC 3195</li> </ul> -<h2>Output Modules</h2> +<a name"om"></a><h2>Output Modules</h2> <p>Output modules process messages. With them, message formats can be transformed and messages be transmitted to various different targets. <ul> @@ -52,14 +60,54 @@ SQLLite, Ingres, Oracle, mSQL)</li> <li><a href="ommail.html">ommail</a> - permits rsyslog to alert folks by mail if something important happens</li> <li><a href="omoracle.html">omoracle</a> - output module for Oracle (native OCI interface)</li> +<li><a href="omudpspoof.html">omudpspoof</a> - output module sending UDP syslog messages with a spoofed address</li> +</ul> + +<a name="pm"></a><h2>Parser Modules</h2> +<p>Parser modules are used to parse message content, once the message has been +received. They can be used to process custom message formats or invalidly formatted +messages. For details, please see the <a href="messageparser.html">rsyslog +message parser documentation</a>. +<p>The current modules are currently provided as part of rsyslog: +<ul> +<li>pmrfc5424 - parses RFC5424-formatted messages (the new syslog standard) +<li>pmrfc3164 - the traditional/legacy syslog parser </ul> -<h2>Library Modules</h2> +<a name="mm"></a><h2>Message Modification Modules</h2> +<p>Message modification modules are used to change the content of messages being processed. +They can be implemented using either the output module or the parser module interface. +From the rsyslog core's point of view, they actually are output or parser modules, it is their +implementation that makes them special. +<p>Currently, there do not exist any such modules, but could be written with +the methods the engine provides. They could be used, for example, to: +<ul> +<li>anonymize message content +<li>add dynamically computed content to message (fields) +</ul> + +<a name="lm"></a><h2>Library Modules</h2> <p>Library modules provide dynamically loadable functionality for parts of rsyslog, most often for other loadable modules. They can not be user-configured and are loaded automatically by some components. They are just mentioned so that error messages that point to library moduls can be understood. No module list is provided. +<h2>Where are the modules integrated into the Message Flow?</h2> +<p>Depending on their module type, modules may access and/or modify messages at +various stages during rsyslog's processing. Note that only the "core type" (e.g. input, +output) but not any type derived from it (message modification module) specifies when +a module is called. +<p>The simplified workflow is as follows: +<p align="center"> +<img src="module_workflow.png" alt"rsyslog: loadable modules and message flow"> +<p>As can be seen, messages are received by input modules, then passed to one or many +parser modules, which generate the in-memory representation of the message and may +also modify the message itself. The, the internal representation is passed to +output modules, which may output a message and (with the interfaces newly introduced +in v5) may also modify messageo object content. +<p>Note that the actual flow is much more complex and depends a lot on queue and +filter settings. This graphic above is a high-level message flow diagram. + <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> diff --git a/doc/src/module_workflow.dia b/doc/src/module_workflow.dia Binary files differnew file mode 100644 index 00000000..178571f4 --- /dev/null +++ b/doc/src/module_workflow.dia diff --git a/doc/src/tls.dia b/doc/src/tls.dia Binary files differindex 77e5d185..d7c9811d 100644 --- a/doc/src/tls.dia +++ b/doc/src/tls.dia diff --git a/doc/status.html b/doc/status.html index ff056489..cbe15297 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,31 +2,28 @@ <html><head><title>rsyslog status page</title></head> <body> <h2>rsyslog status page</h2> -<p>This page reflects the status as of 2009-11-05.</p> +<p>This page reflects the status as of 2010-01-13.</p> <h2>Current Releases</h2> -<p><b>v5 development:</b> 5.3.4 [2009-11-04] - -<a href="http://www.rsyslog.com/Article423.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-185.phtml">download</a> -<br> +<p><b>v5 development:</b> 5.5.1 [2009-11-27] - +<a href="http://www.rsyslog.com/Article433.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-190.phtml">download</a> <!-- not at the moment! +<br> <b>v4 development:</b> 4.5.1 [2009-07-15] - <a href="http://www.rsyslog.com/Article388.phtml">change log</a> - <a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-167.phtml">download</a></p> --> -<!-- not at the moment! -<br><b>v5-beta:</b> 5.1.6 [2009-10-15] - -<a href="http://www.rsyslog.com/Article413.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-180.phtml">download</a> ---> +<p><b>v5-beta:</b> 5.3.6 [2010-01-13] - +<a href="http://www.rsyslog.com/Article435.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-191.phtml">download</a> -<br><b>v4-beta:</b> 4.5.6 [2009-11-05] - -<a href="http://www.rsyslog.com/Article425.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-186.phtml">download</a></p> +<br><b>v4-beta:</b> 4.5.7 [2009-11-18] - +<a href="http://www.rsyslog.com/Article429.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-188.phtml">download</a></p> -<p><b>v5 stable:</b> 5.2.0 [2009-11-02] (recommended to use -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-183.phtml">5.3.3</a> instead) - +<p><b>v5 stable:</b> 5.2.0 [2009-11-02] (recommended to use v5-beta instead) - <a href="http://www.rsyslog.com/Article421.phtml">change log</a> - <a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-184.phtml">download</a> @@ -52,11 +52,14 @@ #include "errmsg.h" #include "gss-misc.h" #include "debug.h" +#include "glbl.h" +#include "unlimited_select.h" MODULE_TYPE_LIB /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) static void display_status_(char *m, OM_uint32 code, int type) @@ -109,28 +112,38 @@ static int read_all(int fd, char *buf, unsigned int nbyte) { int ret; char *ptr; - fd_set rfds; struct timeval tv; +#ifdef USE_UNLIMITED_SELECT + fd_set *pRfds = malloc(glbl.GetFdSetSize()); +#else + fd_set rfds; + fd_set *pRfds = &rfds; +#endif for (ptr = buf; nbyte; ptr += ret, nbyte -= ret) { - FD_ZERO(&rfds); - FD_SET(fd, &rfds); + FD_ZERO(pRfds); + FD_SET(fd, pRfds); tv.tv_sec = 1; tv.tv_usec = 0; - if ((ret = select(FD_SETSIZE, &rfds, NULL, NULL, &tv)) <= 0 - || !FD_ISSET(fd, &rfds)) + if ((ret = select(FD_SETSIZE, pRfds, NULL, NULL, &tv)) <= 0 + || !FD_ISSET(fd, pRfds)) { + freeFdSet(pRfds); return ret; + } ret = recv(fd, ptr, nbyte, 0); if (ret < 0) { if (errno == EINTR) continue; + freeFdSet(pRfds); return (ret); } else if (ret == 0) { + freeFdSet(pRfds); return (ptr - buf); } } + freeFdSet(pRfds); return (ptr - buf); } @@ -265,6 +278,7 @@ BEGINObjClassExit(gssutil, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END M CODESTARTObjClassExit(gssutil) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); ENDObjClassExit(gssutil) @@ -275,6 +289,7 @@ ENDObjClassExit(gssutil) BEGINAbstractObjClassInit(gssutil, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE class also in END MACRO! */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(gssutil) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 2f7e5fee..81b357ef 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -270,6 +270,11 @@ waitMainQEmpty(tcps_sess_t *pSess) dbgprintf("imdiag sleeping, wait mainq drain, curr size %d\n", iMsgQueueSize); srSleep(0,2); /* wait a little bit */ CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + if(iMsgQueueSize == 0) { + /* verify that queue is still empty (else it could just be a race!) */ + srSleep(1,5); /* wait a little bit */ + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + } } CHKiRet(sendResponse(pSess, "mainqueue empty\n")); diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index 1d4e3b4f..dd3d67e3 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -58,6 +58,7 @@ #include "netstrm.h" #include "glbl.h" #include "debug.h" +#include "unlimited_select.h" MODULE_TYPE_INPUT @@ -178,10 +179,10 @@ isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void *pUsrSrv, void*p pGSess = (gss_sess_t*) pUsrSess; if((pGSrv->allowedMethods & ALLOWEDMETHOD_TCP) && - net.isAllowedSender((uchar*)"TCP", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"TCP", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_TCP; if((pGSrv->allowedMethods & ALLOWEDMETHOD_GSS) && - net.isAllowedSender((uchar*)"GSS", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"GSS", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_GSS; if(allowedMethods && pGSess != NULL) pGSess->allowedMethods = allowedMethods; @@ -417,15 +418,20 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) CHKiRet(netstrm.GetSock(pSess->pStrm, &fdSess)); // TODO: method access! if (allowedMethods & ALLOWEDMETHOD_TCP) { int len; - fd_set fds; struct timeval tv; +#ifdef USE_UNLIMITED_SELECT + fd_set *pFds = malloc(glbl.GetFdSetSize()); +#else + fd_set fds; + fd_set *pFds = &fds; +#endif do { - FD_ZERO(&fds); - FD_SET(fdSess, &fds); + FD_ZERO(pFds); + FD_SET(fdSess, pFds); tv.tv_sec = 1; tv.tv_usec = 0; - ret = select(fdSess + 1, &fds, NULL, NULL, &tv); + ret = select(fdSess + 1, pFds, NULL, NULL, &tv); } while (ret < 0 && errno == EINTR); if (ret < 0) { errmsg.LogError(0, RS_RET_ERR, "TCP session %p will be closed, error ignored\n", pSess); @@ -478,6 +484,8 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) pGSess->allowedMethods = ALLOWEDMETHOD_TCP; ABORT_FINALIZE(RS_RET_OK); // TODO: define good error codes } + + freeFdSet(pFds); } context = &pGSess->gss_context; diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 176b5b18..2348c974 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -97,7 +97,7 @@ static int isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) { - return net.isAllowedSender(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN); + return net.isAllowedSender2(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN, 1); } diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 307b684f..07a07d74 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -63,6 +63,7 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(prop) DEFobjCurrIf(ruleset) +static int bDoACLCheck; /* are ACL checks neeed? Cached once immediately before listener startup */ static int iMaxLine; /* maximum UDP message size supported */ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitted sender was last discarded * This shall prevent remote DoS when the "discard on disallowed sender" @@ -117,7 +118,6 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) if(udpLstnSocks == NULL) { /* esay, we can just replace it */ udpLstnSocks = newSocks; -RUNLOG_VAR("%d", newSocks[0]); CHKmalloc(udpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (newSocks[0] + 1))); for(iDst = 1 ; iDst <= newSocks[0] ; ++iDst) udpRulesets[iDst] = pBindRuleset; @@ -200,7 +200,7 @@ finalize_it: */ static inline rsRetVal processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, - uchar *fromHost, uchar *fromHostFQDN, uchar *fromHostIP, ruleset_t *pRuleset) + ruleset_t *pRuleset) { DEFiRet; int iNbrTimeUsed; @@ -235,37 +235,39 @@ processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, /* if we reach this point, we had a good receive and can process the packet received */ /* check if we have a different sender than before, if so, we need to query some new values */ - if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { - CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); - memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ - /* Here we check if a host is permitted to send us - * syslog messages. If it isn't, we do not further - * process the message but log a warning (if we are - * configured to do this). - * rgerhards, 2005-09-26 - */ - *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", - (struct sockaddr *)&frominet, (char*)fromHostFQDN); - - if(!*pbIsPermitted) { - DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); - if(glbl.GetOption_DisallowWarning) { - time_t tt; - - datetime.GetTime(&tt); - if(tt > ttLastDiscard + 60) { - ttLastDiscard = tt; - errmsg.LogError(0, NO_ERRCODE, - "UDP message from disallowed sender %s discarded", - (char*)fromHost); + if(bDoACLCheck) { + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us syslog messages. If it isn't, + * we do not further process the message but log a warning (if we are + * configured to do this). However, if the check would require name resolution, + * it is postponed to the main queue. See also my blog post at + * http://blog.gerhards.net/2009/11/acls-imudp-and-accepting-messages.html + * rgerhards, 2009-11-16 + */ + *pbIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)&frominet, "", 0); + + if(*pbIsPermitted == 0) { + DBGPRINTF("msg is not from an allowed sender\n"); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender discarded"); + } } } } + } else { + *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d)/%s,acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, fromHost, *pbIsPermitted, pRcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); - if(*pbIsPermitted) { + if(*pbIsPermitted != 0) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); } @@ -275,9 +277,10 @@ processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, MsgSetInputName(pMsg, pInputName); MsgSetRuleset(pMsg, pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); - CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; + if(*pbIsPermitted == 2) + pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */ + CHKiRet(msgSetFromSockinfo(pMsg, &frominet)); CHKiRet(submitMsg(pMsg)); } } @@ -307,9 +310,6 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) int i; struct sockaddr_storage frominetPrev; int bIsPermitted; - uchar fromHost[NI_MAXHOST]; - uchar fromHostIP[NI_MAXHOST]; - uchar fromHostFQDN[NI_MAXHOST]; struct epoll_event *udpEPollEvt = NULL; struct epoll_event currEvt[NUM_EPOLL_EVENTS]; char errStr[1024]; @@ -359,7 +359,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) for(i = 0 ; i < nfds ; ++i) { processSocket(pThrd, udpLstnSocks[currEvt[i].data.u64], &frominetPrev, &bIsPermitted, - fromHost, fromHostFQDN, fromHostIP, udpRulesets[currEvt[i].data.u64]); + udpRulesets[currEvt[i].data.u64]); } } @@ -377,7 +377,6 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) int maxfds; int nfds; int i; - fd_set readfds; struct sockaddr_storage frominetPrev; int bIsPermitted; uchar fromHost[NI_MAXHOST]; @@ -399,21 +398,21 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) * is given without -a, we do not need to listen at all.. */ maxfds = 0; - FD_ZERO (&readfds); + FD_ZERO (pReadfds); /* Add the UDP listen sockets to the list of read descriptors. */ for (i = 0; i < *udpLstnSocks; i++) { if (udpLstnSocks[i+1] != -1) { if(Debug) net.debugListenInfo(udpLstnSocks[i+1], "UDP"); - FD_SET(udpLstnSocks[i+1], &readfds); + FD_SET(udpLstnSocks[i+1], pReadfds); if(udpLstnSocks[i+1]>maxfds) maxfds=udpLstnSocks[i+1]; } } if(Debug) { dbgprintf("--------imUDP calling select, active file descriptors (max %d): ", maxfds); for (nfds = 0; nfds <= maxfds; ++nfds) - if ( FD_ISSET(nfds, &readfds) ) + if ( FD_ISSET(nfds, pReadfds) ) dbgprintf("%d ", nfds); dbgprintf("\n"); } @@ -443,10 +442,6 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) */ BEGINrunInput CODESTARTrunInput - /* this is an endless loop - it is terminated when the thread is - * signalled to do so. This, however, is handled by the framework, - * right into the sleep below. - */ iRet = rcvMainLoop(pThrd); ENDrunInput @@ -460,6 +455,7 @@ CODESTARTwillRun CHKiRet(prop.ConstructFinalize(pInputName)); net.PrintAllowedSenders(1); /* UDP */ + net.HasRestrictions(UCHAR_CONSTANT("UDP"), &bDoACLCheck); /* UDP */ /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index cf6df56c..06f9caad 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -46,6 +46,7 @@ #include "msg.h" #include "prop.h" #include "debug.h" +#include "unlimited_select.h" MODULE_TYPE_INPUT @@ -294,7 +295,13 @@ BEGINrunInput int nfds; int i; int fd; - fd_set readfds; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = malloc(glbl.GetFdSetSize()); +#else + fd_set readfds; + fd_set *pReadfds = &readfds; +#endif + CODESTARTrunInput /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, @@ -308,11 +315,11 @@ CODESTARTrunInput * is given without -a, we do not need to listen at all.. */ maxfds = 0; - FD_ZERO (&readfds); + FD_ZERO (pReadfds); /* Copy master connections */ for (i = startIndexUxLocalSockets; i < nfunix; i++) { if (funix[i] != -1) { - FD_SET(funix[i], &readfds); + FD_SET(funix[i], pReadfds); if (funix[i]>maxfds) maxfds=funix[i]; } } @@ -320,20 +327,20 @@ CODESTARTrunInput if(Debug) { dbgprintf("--------imuxsock calling select, active file descriptors (max %d): ", maxfds); for (nfds= 0; nfds <= maxfds; ++nfds) - if ( FD_ISSET(nfds, &readfds) ) + if ( FD_ISSET(nfds, pReadfds) ) dbgprintf("%d ", nfds); dbgprintf("\n"); } /* wait for io to become ready */ - nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); + nfds = select(maxfds+1, (fd_set *) pReadfds, NULL, NULL, NULL); if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ for (i = 0; i < nfunix && nfds > 0; i++) { if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ - if ((fd = funix[i]) != -1 && FD_ISSET(fd, &readfds)) { + if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) { readSocket(fd, i); --nfds; /* indicate we have processed one */ } @@ -341,6 +348,7 @@ CODESTARTrunInput } finalize_it: + freeFdSet(pReadfds); RETiRet; ENDrunInput diff --git a/plugins/omdbalerting/Makefile.am b/plugins/omdbalerting/Makefile.am new file mode 100644 index 00000000..becf29b0 --- /dev/null +++ b/plugins/omdbalerting/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omdbalerting.la + +omdbalerting_la_SOURCES = omdbalerting.c +omdbalerting_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omdbalerting_la_LDFLAGS = -module -avoid-version +omdbalerting_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omdbalerting/omdbalerting.c b/plugins/omdbalerting/omdbalerting.c new file mode 100644 index 00000000..2e04391c --- /dev/null +++ b/plugins/omdbalerting/omdbalerting.c @@ -0,0 +1,144 @@ +/* omdbalerting.c + * generate alerts based on database contents - so far a skeleton + * left for implementation by somebody else (skeleton created on request). + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-11-17 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* config variables */ + + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omdbalerting:", sizeof(":dbalerting:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omdbalerting:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + /* we request the standard interface via template, others may be more useful + * here. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + // SAMPLE! CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomdbalertingensurelfending", 0, eCmdHdlrBinary, NULL, + // &bEnsureLFEnding, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index caf7c5ca..9047c83d 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -16,6 +16,7 @@ librsyslog_la_SOURCES = \ nsd.h \ glbl.h \ glbl.c \ + unlimited_select.h \ conf.c \ conf.h \ parser.h \ @@ -136,7 +137,10 @@ lmnet_la_LDFLAGS = -module -avoid-version lmnet_la_LIBADD = # network stream master class and stream factory -lmnetstrms_la_SOURCES = netstrms.c netstrms.h netstrm.c netstrm.h nssel.c nssel.h +lmnetstrms_la_SOURCES = netstrms.c netstrms.h \ + netstrm.c netstrm.h \ + nssel.c nssel.h \ + nspoll.c nspoll.h lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version lmnetstrms_la_LIBADD = @@ -152,7 +156,9 @@ lmstrmsrv_la_LIBADD = # plain tcp driver - main driver pkglib_LTLIBRARIES += lmnsd_ptcp.la -lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h nsdsel_ptcp.c nsdsel_ptcp.h +lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h \ + nsdsel_ptcp.c nsdsel_ptcp.h \ + nsdpoll_ptcp.c nsdpoll_ptcp.h lmnsd_ptcp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnsd_ptcp_la_LDFLAGS = -module -avoid-version lmnsd_ptcp_la_LIBADD = diff --git a/runtime/debug.c b/runtime/debug.c index 545ac876..a517b1ba 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1308,11 +1308,11 @@ dbgGetRuntimeOptions(void) /* this is earlier in the process than the -d option, as such it * allows us to spit out debug messages from the very beginning. */ - Debug = 1; + Debug = DEBUG_FULL; debugging_on = 1; } else if(!strcasecmp((char*)optname, "debugondemand")) { /* Enables debugging, but turns off debug output */ - Debug = 1; + Debug = DEBUG_ONDEMAND; debugging_on = 1; dbgprintf("Note: debug on demand turned on via configuraton file, " "use USR1 signal to activate.\n"); diff --git a/runtime/debug.h b/runtime/debug.h index 8d9c1ceb..c011dd2d 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -29,6 +29,11 @@ #include <pthread.h> #include "obj-types.h" +/* some settings for various debug modes */ +#define DEBUG_OFF 0 +#define DEBUG_ONDEMAND 1 +#define DEBUG_FULL 2 + /* external static data elements (some time to be replaced) */ extern int Debug; /* debug flag - read-only after startup */ extern int debugging_on; /* read-only, except on sig USR1 */ diff --git a/runtime/glbl.c b/runtime/glbl.c index 71c2ed0d..ac08791f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -74,6 +74,9 @@ static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm d static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */ static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */ static int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ +#ifdef USE_UNLIMITED_SELECT +static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */ +#endif /* define a macro for the simple properties' set and get functions @@ -106,6 +109,9 @@ SIMP_PROP(DisableDNS, bDisableDNS, int) SIMP_PROP(LocalDomain, LocalDomain, uchar*) SIMP_PROP(StripDomains, StripDomains, char**) SIMP_PROP(LocalHosts, LocalHosts, char**) +#ifdef USE_UNLIMITED_SELECT +SIMP_PROP(FdSetSize, iFdSetSize, int) +#endif SIMP_PROP_SET(LocalFQDNName, LocalFQDNName, uchar*) SIMP_PROP_SET(LocalHostName, LocalHostName, uchar*) @@ -284,6 +290,9 @@ CODESTARTobjQueryInterface(glbl) SIMP_PROP(DfltNetstrmDrvrCAF) SIMP_PROP(DfltNetstrmDrvrKeyFile) SIMP_PROP(DfltNetstrmDrvrCertFile) +#ifdef USE_UNLIMITED_SELECT + SIMP_PROP(FdSetSize) +#endif #undef SIMP_PROP finalize_it: ENDobjQueryInterface(glbl) @@ -317,6 +326,9 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; bPreserveFQDN = 0; +#ifdef USE_UNLIMITED_SELECT + iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); +#endif return RS_RET_OK; } diff --git a/runtime/glbl.h b/runtime/glbl.h index 7506f16b..4b4bdf83 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -66,9 +66,20 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ void (*SetGlobalInputTermination)(void); /* added v5, 2009-11-03 */ SIMP_PROP(ParseHOSTNAMEandTAG, int) + /* note: v4, v5 are already used by more recent versions, so we need to skip them! */ + /* added v6, 2009-11-16 as part of varmojfekoj's "unlimited select()" patch + * Note that it must be always present, otherwise the interface would have different + * versions depending on compile settings, what is not acceptable. + * Use this property with care, it is only truly available if UNLIMITED_SELECT is enabled + * (I did not yet further investigate the details, because that code hopefully can be removed + * at some later stage). + */ + SIMP_PROP(FdSetSize, int) + /* v7: was neeeded to mean v5+v6 - do NOT add anything else for that version! */ + /* next change is v8! */ #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ diff --git a/runtime/modules.c b/runtime/modules.c index fd3468d8..1af94abc 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -472,7 +472,6 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction); if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) { pNew->mod.om.endTransaction = dummyEndTransaction; - //pNew->mod.om.beginTransaction = dummyEndTransaction; } else if(localRet != RS_RET_OK) { ABORT_FINALIZE(localRet); } @@ -559,10 +558,35 @@ static void modPrintList(void) dbgprintf(" module.\n"); dbgprintf("Entry points:\n"); dbgprintf("\tqueryEtryPt: 0x%lx\n", (unsigned long) pMod->modQueryEtryPt); - dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); - dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); dbgprintf("\tdbgPrintInstInfo: 0x%lx\n", (unsigned long) pMod->dbgPrintInstInfo); dbgprintf("\tfreeInstance: 0x%lx\n", (unsigned long) pMod->freeInstance); + switch(pMod->eType) { + case eMOD_OUT: + dbgprintf("Output Module Entry Points:\n"); + dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); + dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); + dbgprintf("\ttryResume: 0x%lx\n", (unsigned long) pMod->tryResume); + dbgprintf("\tdoHUP: 0x%lx\n", (unsigned long) pMod->doHUP); + dbgprintf("\tBeginTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.beginTransaction == dummyBeginTransaction) ? + 0 : pMod->mod.om.beginTransaction)); + dbgprintf("\tEndTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.endTransaction == dummyEndTransaction) ? + 0 : pMod->mod.om.endTransaction)); + break; + case eMOD_IN: + dbgprintf("Input Module Entry Points\n"); + dbgprintf("\trunInput: 0x%lx\n", (unsigned long) pMod->mod.im.runInput); + dbgprintf("\twillRun: 0x%lx\n", (unsigned long) pMod->mod.im.willRun); + dbgprintf("\tafterRun: 0x%lx\n", (unsigned long) pMod->mod.im.afterRun); + break; + case eMOD_LIB: + break; + case eMOD_PARSER: + dbgprintf("Parser Module Entry Points\n"); + dbgprintf("\tparse: 0x%lx\n", (unsigned long) pMod->mod.pm.parse); + break; + } dbgprintf("\n"); pMod = GetNxt(pMod); /* done, go next */ } diff --git a/runtime/msg.c b/runtime/msg.c index cc2f0b69..755f78cb 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -35,6 +35,8 @@ #include <string.h> #include <assert.h> #include <ctype.h> +#include <sys/socket.h> +#include <netdb.h> #if HAVE_MALLOC_H # include <malloc.h> #endif @@ -51,6 +53,7 @@ #include "unicode-helper.h" #include "ruleset.h" #include "prop.h" +#include "net.h" /* static data */ DEFobjStaticHelpers @@ -59,6 +62,7 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) DEFobjCurrIf(prop) +DEFobjCurrIf(net) static struct { uchar *pszName; @@ -284,6 +288,41 @@ static inline int getProtocolVersion(msg_t *pM) } +/* do a DNS reverse resolution, if not already done, reflect status + * rgerhards, 2009-11-16 + */ +static inline rsRetVal +resolveDNS(msg_t *pMsg) { + rsRetVal localRet; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + DEFiRet; + + CHKiRet(objUse(net, CORE_COMPONENT)); + if(pMsg->msgFlags & NEEDS_DNSRESOL) { + localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); + if(localRet == RS_RET_OK) { + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + } + } +finalize_it: + if(iRet != RS_RET_OK) { + /* best we can do: remove property */ + MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); + prop.Destruct(&propFromHost); + } + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; +} + + static inline void getInputName(msg_t *pM, uchar **ppsz, int *plen) { @@ -307,6 +346,7 @@ getRcvFromIP(msg_t *pM) if(pM == NULL) { psz = UCHAR_CONSTANT(""); } else { + resolveDNS(pM); /* make sure we have a resolved entry */ if(pM->pRcvFromIP == NULL) psz = UCHAR_CONSTANT(""); else @@ -660,7 +700,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pCSMSGID = NULL; pM->pInputName = NULL; pM->pRcvFromIP = NULL; - pM->pRcvFrom = NULL; + pM->rcvFrom.pRcvFrom = NULL; pM->pRuleset = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); @@ -761,8 +801,12 @@ CODESTARTobjDestruct(msg) freeHOSTNAME(pThis); if(pThis->pInputName != NULL) prop.Destruct(&pThis->pInputName); - if(pThis->pRcvFrom != NULL) - prop.Destruct(&pThis->pRcvFrom); + if((pThis->msgFlags & NEEDS_DNSRESOL) == 0) { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } else { + free(pThis->rcvFrom.pfrominet); + } if(pThis->pRcvFromIP != NULL) prop.Destruct(&pThis->pRcvFromIP); free(pThis->pszRcvdAt3164); @@ -848,6 +892,7 @@ ENDobjDestruct(msg) msg_t* MsgDup(msg_t* pOld) { msg_t* pNew; + rsRetVal localRet; assert(pOld != NULL); @@ -868,9 +913,19 @@ msg_t* MsgDup(msg_t* pOld) pNew->iLenMSG = pOld->iLenMSG; pNew->iLenTAG = pOld->iLenTAG; pNew->iLenHOSTNAME = pOld->iLenHOSTNAME; - if(pOld->pRcvFrom != NULL) { - pNew->pRcvFrom = pOld->pRcvFrom; - prop.AddRef(pNew->pRcvFrom); + if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) { + localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet); + if(localRet != RS_RET_OK) { + /* if something fails, we accept loss of this property, it is + * better than losing the whole message. + */ + pNew->msgFlags &= ~NEEDS_DNSRESOL; + } + } else { + if(pOld->rcvFrom.pRcvFrom != NULL) { + pNew->rcvFrom.pRcvFrom = pOld->rcvFrom.pRcvFrom; + prop.AddRef(pNew->rcvFrom.pRcvFrom); + } } if(pOld->pRcvFromIP != NULL) { pNew->pRcvFromIP = pOld->pRcvFromIP; @@ -1648,12 +1703,13 @@ int getHOSTNAMELen(msg_t *pM) if(pM == NULL) return 0; else - if(pM->pszHOSTNAME == NULL) - if(pM->pRcvFrom == NULL) + if(pM->pszHOSTNAME == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) return 0; else - return prop.GetStringLen(pM->pRcvFrom); - else + return prop.GetStringLen(pM->rcvFrom.pRcvFrom); + } else return pM->iLenHOSTNAME; } @@ -1664,12 +1720,13 @@ char *getHOSTNAME(msg_t *pM) return ""; else if(pM->pszHOSTNAME == NULL) { - if(pM->pRcvFrom == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) { return ""; } else { uchar *psz; int len; - prop.GetString(pM->pRcvFrom, &psz, &len); + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); return (char*) psz; } } else { @@ -1683,13 +1740,15 @@ uchar *getRcvFrom(msg_t *pM) uchar *psz; int len; BEGINfunc + if(pM == NULL) { psz = UCHAR_CONSTANT(""); } else { - if(pM->pRcvFrom == NULL) + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) psz = UCHAR_CONSTANT(""); else - prop.GetString(pM->pRcvFrom, &psz, &len); + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); } ENDfunc return psz; @@ -1845,6 +1904,28 @@ void MsgSetInputName(msg_t *pThis, prop_t *inputName) } +/* Set the pfrominet socket store, so that we can obtain the peer at some + * later time. Note that we do not check if pRcvFrom is already set, so this + * function must only be called during message creation. + * NOTE: msgFlags is NOT set. While this is somewhat a violation of layers, + * it is done because it gains us some performance. So the caller must make + * sure the message flags are properly maintained. For all current callers, + * this is always the case and without extra effort required. + * rgerhards, 2009-11-17 + */ +rsRetVal +msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa){ + DEFiRet; + assert(pThis->rcvFrom.pRcvFrom == NULL); + + CHKmalloc(pThis->rcvFrom.pfrominet = malloc(sizeof(struct sockaddr_storage))); + memcpy(pThis->rcvFrom.pfrominet, sa, sizeof(struct sockaddr_storage)); + +finalize_it: + RETiRet; +} + + /* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef() * on the property, because this must be done in all current cases and there * is no case expected where this may not be necessary. @@ -1855,9 +1936,15 @@ void MsgSetRcvFrom(msg_t *pThis, prop_t *new) assert(pThis != NULL); prop.AddRef(new); - if(pThis->pRcvFrom != NULL) - prop.Destruct(&pThis->pRcvFrom); - pThis->pRcvFrom = new; + if(pThis->msgFlags & NEEDS_DNSRESOL) { + if(pThis->rcvFrom.pfrominet != NULL) + free(pThis->rcvFrom.pfrominet); + pThis->msgFlags &= ~NEEDS_DNSRESOL; + } else { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } + pThis->rcvFrom.pRcvFrom = new; } diff --git a/runtime/msg.h b/runtime/msg.h index 9101cef7..366dce64 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -61,12 +61,6 @@ struct msg { pthread_mutex_t mut; bool bDoLock; /* use the mutex? */ short iRefCount; /* reference counter (0 = unused) */ - /* background: the hostname is not present on "regular" messages - * received via UNIX domain sockets from the same machine. However, - * it is available when we have a forwarder (e.g. rfc3195d) using local - * sockets. All in all, the parser would need parse templates, that would - * resolve all these issues... rgerhards, 2005-10-06 - */ short iSeverity; /* the severity 0..7 */ short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ @@ -94,8 +88,12 @@ struct msg { cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ prop_t *pInputName; /* input name property */ - prop_t *pRcvFrom; /* name of system message was received from */ prop_t *pRcvFromIP; /* IP of system message was received from */ + union { + prop_t *pRcvFrom;/* name of system message was received from */ + struct sockaddr_storage *pfrominet; /* unresolved name */ + } rcvFrom; + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -129,6 +127,8 @@ struct msg { #define MARK 0x008 /* this message is a mark */ #define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ #define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ +#define NEEDS_DNSRESOL 0x040 /* fromhost address is unresolved and must be locked up via DNS reverse lookup first */ +#define NEEDS_ACLCHK_U 0x080 /* check UDP ACLs after DNS resolution has been done in main queue consumer */ /* function prototypes @@ -148,6 +148,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); +rsRetVal msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa); void MsgSetRcvFrom(msg_t *pMsg, prop_t*); void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); diff --git a/runtime/net.c b/runtime/net.c index f52d408c..ab431f7c 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -892,15 +892,18 @@ rsRetVal addAllowedSenderLine(char* pName, uchar** ppRestOfConfLine) * including IPv4/v6 as well as domain name wildcards. * This is a helper to isAllowedSender. As it is only called once, it is * declared inline. - * Returns 0 if they do not match, something else otherwise. - * contributed 1007-07-16 by mildew@gmail.com + * Returns 0 if they do not match, 1 if they match and 2 if a DNS name would have been required. + * contributed 2007-07-16 by mildew@gmail.com */ -static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost) +static inline int +MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { assert(pAllow != NULL); assert(pFrom != NULL); if(F_ISSET(pAllow->flags, ADDR_NAME)) { + if(bChkDNS == 0) + return 2; dbgprintf("MaskCmp: host=\"%s\"; pattern=\"%s\"\n", pszFromHost, pAllow->addr.HostWildcard); # if !defined(FNM_CASEFOLD) @@ -967,18 +970,22 @@ static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr /* check if a sender is allowed. The root of the the allowed sender. * list must be proveded by the caller. As such, this function can be * used to check both UDP and TCP allowed sender lists. - * returns 1, if the sender is allowed, 0 otherwise. + * returns 1, if the sender is allowed, 0 if not and 2 if we could not + * obtain a result because we would need a dns name, which we don't have + * (2 was added rgerhards, 2009-11-16). * rgerhards, 2005-09-26 */ -static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) +static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { struct AllowedSenders *pAllow; struct AllowedSenders *pAllowRoot; + int bNeededDNS = 0; /* partial check because we could not resolve DNS? */ + int ret; assert(pFrom != NULL); if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK) - return 0; /* if something went wrong, we denie access - that's the better choice... */ + return 0; /* if something went wrong, we deny access - that's the better choice... */ if(pAllowRoot == NULL) return 1; /* checking disabled, everything is valid! */ @@ -990,10 +997,20 @@ static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *p * that the sender is disallowed. */ for(pAllow = pAllowRoot ; pAllow != NULL ; pAllow = pAllow->pNext) { - if (MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost)) + ret = MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost, bChkDNS); + if(ret == 1) return 1; + else if(ret == 2) + bNeededDNS = 2; } - return 0; + return bNeededDNS; +} + + +/* legacy API, not to be used any longer */ +static int +isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) { + return isAllowedSender2(pszType, pFrom, pszFromHost, 1); } @@ -1533,12 +1550,36 @@ static int CmpHost(struct sockaddr_storage *s1, struct sockaddr_storage* s2, siz ret = memcmp(s1, s2, socklen); } -dbgprintf("CmpHost returns %d\n", ret); finalize_it: return ret; } + +/* check if restrictions (ALCs) exists. The goal of this function is to disable the + * somewhat time-consuming ACL checks if no restrictions are defined (the usual case). + * This also permits to gain some speedup by using firewall-based ACLs instead of + * rsyslog ACLs (the recommended method. + * rgerhards, 2009-11-16 + */ +static rsRetVal +HasRestrictions(uchar *pszType, int *bHasRestrictions) { + struct AllowedSenders *pAllowRoot; + DEFiRet; + + CHKiRet(setAllowRoot(&pAllowRoot, pszType)); + + *bHasRestrictions = (pAllowRoot == NULL) ? 0 : 1; + +finalize_it: + if(iRet != RS_RET_OK) { + *bHasRestrictions = 1; /* in this case it is better to check individually */ + DBGPRINTF("Error %d trying to obtain ACL restriction state of '%s'\n", iRet, pszType); + } + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-03-05 */ @@ -1562,12 +1603,14 @@ CODESTARTobjQueryInterface(net) pIf->create_udp_socket = create_udp_socket; pIf->closeUDPListenSockets = closeUDPListenSockets; pIf->isAllowedSender = isAllowedSender; + pIf->isAllowedSender2 = isAllowedSender2; pIf->should_use_so_bsdcompat = should_use_so_bsdcompat; pIf->getLocalHostname = getLocalHostname; pIf->AddPermittedPeer = AddPermittedPeer; pIf->DestructPermittedPeers = DestructPermittedPeers; pIf->PermittedPeerWildcardMatch = PermittedPeerWildcardMatch; pIf->CmpHost = CmpHost; + pIf->HasRestrictions = HasRestrictions; /* data members */ pIf->pACLAddHostnameOnFail = &ACLAddHostnameOnFail; pIf->pACLDontResolve = &ACLDontResolve; diff --git a/runtime/net.h b/runtime/net.h index ec364b1c..101ce79d 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -139,7 +139,7 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ void (*debugListenInfo)(int fd, char *type); int *(*create_udp_socket)(uchar *hostname, uchar *LogPort, int bIsServer); void (*closeUDPListenSockets)(int *finet); - int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); + int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); /* deprecated! */ rsRetVal (*getLocalHostname)(uchar**); int (*should_use_so_bsdcompat)(void); /* permitted peer handling should be replaced by something better (see comments above) */ @@ -148,11 +148,14 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ rsRetVal (*PermittedPeerWildcardMatch)(permittedPeers_t *pPeer, uchar *pszNameToMatch, int *pbIsMatching); /* v5 interface additions */ int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t); + /* v6 interface additions - 2009-11-16 */ + rsRetVal (*HasRestrictions)(uchar *, int *bHasRestrictions); + int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS); /* data members - these should go away over time... TODO */ int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ ENDinterface(net) -#define netCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define netCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* prototypes */ PROTOTYPEObj(net); diff --git a/runtime/netstrms.c b/runtime/netstrms.c index 6b28e7ea..e9ff2568 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -36,6 +36,7 @@ #include "nsd.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "netstrms.h" MODULE_TYPE_LIB @@ -304,6 +305,7 @@ ENDObjClassInit(netstrms) BEGINmodExit CODESTARTmodExit nsselClassExit(); + nspollClassExit(); netstrmsClassExit(); netstrmClassExit(); /* we use this object, so we must exit it after we are finished */ ENDmodExit @@ -322,6 +324,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(netstrmClassInit(pModInfo)); CHKiRet(nsselClassInit(pModInfo)); + CHKiRet(nspollClassInit(pModInfo)); CHKiRet(netstrmsClassInit(pModInfo)); ENDmodInit /* vi:set ai: diff --git a/runtime/nsd.h b/runtime/nsd.h index 8668c934..e5b9320b 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -87,4 +87,13 @@ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ ENDinterface(nsdsel) #define nsdselCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface for the epoll call */ +BEGINinterface(nsdpoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nsdpoll_t **ppThis); + rsRetVal (*Destruct)(nsdpoll_t **ppThis); + rsRetVal (*Ctl)(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op); + rsRetVal (*Wait)(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); +ENDinterface(nsdpoll) +#define nsdpollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + #endif /* #ifndef INCLUDED_NSD_H */ diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index fe31ab40..744955c7 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -48,6 +48,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nsdsel_ptcp.h" +#include "nsdpoll_ptcp.h" #include "nsd_ptcp.h" MODULE_TYPE_LIB @@ -562,6 +563,7 @@ finalize_it: static rsRetVal Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) { + char errStr[1024]; DEFiRet; nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; ISOBJ_TYPE_assert(pThis, nsd_ptcp); @@ -571,7 +573,9 @@ Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) if(*pLenBuf == 0) { ABORT_FINALIZE(RS_RET_CLOSED); } else if (*pLenBuf < 0) { - ABORT_FINALIZE(RS_RET_ERR); + rs_strerror_r(errno, errStr, sizeof(errStr)); + dbgprintf("error during recv on NSD %p: %s\n", pNsd, errStr); + ABORT_FINALIZE(RS_RET_RCV_ERR); } finalize_it: @@ -821,6 +825,7 @@ ENDObjClassInit(nsd_ptcp) BEGINmodExit CODESTARTmodExit + nsdpoll_ptcpClassExit(); nsdsel_ptcpClassExit(); nsd_ptcpClassExit(); ENDmodExit @@ -839,6 +844,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(nsd_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ CHKiRet(nsdsel_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + CHKiRet(nsdpoll_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ ENDmodInit /* vi:set ai: */ diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c new file mode 100644 index 00000000..85aac04c --- /dev/null +++ b/runtime/nsdpoll_ptcp.c @@ -0,0 +1,284 @@ +/* nsdpoll_ptcp.c + * + * An implementation of the nsd epoll() interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "errmsg.h" +#include "srUtils.h" +#include "nspoll.h" +#include "nsd_ptcp.h" +#include "nsdpoll_ptcp.h" +#include "unlimited_select.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + + +/* -START------------------------- helpers for event list ------------------------------------ */ + +/* add new entry to list. We assume that the fd is not already present and DO NOT check this! + * Returns newly created entry in pEvtLst. + * Note that we currently need to use level-triggered mode, because the upper layers do not work + * in parallel. As such, in edge-triggered mode we may not get notified, because new data comes + * in after we have read everything that was present. To use ET mode, we need to change the upper + * peers so that they immediately start a new wait before processing the data read. That obviously + * requires more elaborate redesign and we postpone this until the current more simplictic mode has + * been proven OK in practice. + * rgerhards, 2009-11-18 + */ +static inline rsRetVal +addEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, int mode, nsd_ptcp_t *pSock, nsdpoll_epollevt_lst_t **pEvtLst) { + nsdpoll_epollevt_lst_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (nsdpoll_epollevt_lst_t*) malloc(sizeof(nsdpoll_epollevt_lst_t))); + pNew->id = id; + pNew->pUsr = pUsr; + pNew->pSock = pSock; + pNew->event.events = 0; /* TODO: at some time we should be able to use EPOLLET */ + if(mode & NSDPOLL_IN) + pNew->event.events |= EPOLLIN; + if(mode & NSDPOLL_OUT) + pNew->event.events |= EPOLLOUT; + pNew->event.data.u64 = (uint64) pNew; + pNew->pNext = pThis->pRoot; + pThis->pRoot = pNew; + *pEvtLst = pNew; + +finalize_it: + RETiRet; +} + + +/* find and unlink the entry identified by id/pUsr from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +unlinkEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, nsdpoll_epollevt_lst_t **ppEvtLst) { + nsdpoll_epollevt_lst_t *pEvtLst; + nsdpoll_epollevt_lst_t *pPrev = NULL; + DEFiRet; + + pEvtLst = pThis->pRoot; + while(pEvtLst != NULL && !(pEvtLst->id == id && pEvtLst->pUsr == pUsr)) { + pPrev = pEvtLst; + pEvtLst = pEvtLst->pNext; + } + if(pEvtLst == NULL) + ABORT_FINALIZE(RS_RET_NOT_FOUND); + + *ppEvtLst = pEvtLst; + + /* unlink */ + if(pPrev == NULL) + pThis->pRoot = pEvtLst->pNext; + else + pPrev->pNext = pEvtLst->pNext; + +finalize_it: + RETiRet; +} + + +/* destruct the provided element. It must already be unlinked from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) { + DEFiRet; + free(*ppEvtLst); + *ppEvtLst = NULL; + RETiRet; +} + + +/* -END--------------------------- helpers for event list ------------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */ +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); + pThis->efd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n"); + pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */ +# endif + if(pThis->efd < 0) { + DBGPRINTF("epoll_create1() could not create fd\n"); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } +finalize_it: +ENDobjConstruct(nsdpoll_ptcp) + + +/* destructor for the nsdpoll_ptcp object */ +BEGINobjDestruct(nsdpoll_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nsdpoll_ptcp) +ENDobjDestruct(nsdpoll_ptcp) + + +/* Modify socket set */ +static rsRetVal +Ctl(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; + nsdpoll_epollevt_lst_t *pEventLst; + int errSave; + char errStr[512]; + DEFiRet; + + if(op == NSDPOLL_ADD) { + dbgprintf("adding nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(addEvent(pThis, id, pUsr, mode, pSock, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_ADD, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + } + } else if(op == NSDPOLL_DEL) { + dbgprintf("removing nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(unlinkEvent(pThis, id, pUsr, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_DEL, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + ABORT_FINALIZE(RS_RET_ERR_EPOLL_CTL); + } + CHKiRet(delEvent(&pEventLst)); + } else { + dbgprintf("program error: invalid NSDPOLL_mode %d - ignoring request\n", op); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Wait for io to become ready. After the successful call, idRdy contains the + * id set by the caller for that i/o event, ppUsr is a pointer to a location + * where the user pointer shall be stored. + * TODO: this is a trivial implementation that only polls one event at a time. We + * may later extend it to poll for multiple events, what would cause less + * overhead. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Wait(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsdpoll_epollevt_lst_t *pOurEvt; + struct epoll_event event; + int nfds; + DEFiRet; + + assert(idRdy != NULL); + assert(ppUsr != NULL); + + nfds = epoll_wait(pThis->efd, &event, 1, timeout); + if(nfds == -1) { + if(errno == EINTR) { + ABORT_FINALIZE(RS_RET_EINTR); + } else { + DBGPRINTF("epoll() returned with error code %d\n", errno); + ABORT_FINALIZE(RS_RET_ERR_EPOLL); + } + } else if(nfds == 0) { + ABORT_FINALIZE(RS_RET_TIMEOUT); + } + + /* we got a valid event, so tell the caller... */ + pOurEvt = (nsdpoll_epollevt_lst_t*) event.data.u64; + *idRdy = pOurEvt->id; + *ppUsr = pOurEvt->pUsr; + +finalize_it: + RETiRet; +} + + +/* ------------------------------ end support for the epoll() interface ------------------------------ */ + + +/* queryInterface function */ +BEGINobjQueryInterface(nsdpoll_ptcp) +CODESTARTobjQueryInterface(nsdpoll_ptcp) + if(pIf->ifVersion != nsdCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpConstruct; + pIf->Destruct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpDestruct; + pIf->Ctl = Ctl; + pIf->Wait = Wait; +finalize_it: +ENDobjQueryInterface(nsdpoll_ptcp) + + +/* exit our class + */ +BEGINObjClassExit(nsdpoll_ptcp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdpoll_ptcp) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(nsdpoll_ptcp) + + +/* Initialize the nsdpoll_ptcp class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nsdpoll_ptcp, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nsdpoll_ptcp) +/* vi:set ai: + */ diff --git a/runtime/nsdpoll_ptcp.h b/runtime/nsdpoll_ptcp.h new file mode 100644 index 00000000..0708e489 --- /dev/null +++ b/runtime/nsdpoll_ptcp.h @@ -0,0 +1,60 @@ +/* An implementation of the nsd poll interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSDPOLL_PTCP_H +#define INCLUDED_NSDPOLL_PTCP_H + +#include "nsd.h" +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#else + typedef void epoll_event_t; +#endif +typedef nsdpoll_if_t nsdpoll_ptcp_if_t; /* we just *implement* this interface */ +/* a helper object to keep track of the epoll event records + * Note that we need to keep track of that list because we need to + * free the events when they are no longer needed. + */ +typedef struct nsdpoll_epollevt_lst_s nsdpoll_epollevt_lst_t; +struct nsdpoll_epollevt_lst_s { + epoll_event_t event; + int id; + void *pUsr; + nsd_ptcp_t *pSock; /* our associated netstream driver data */ + nsdpoll_epollevt_lst_t *pNext; +}; + +/* the nsdpoll_ptcp object */ +struct nsdpoll_ptcp_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int efd; /* file descriptor used by epoll */ + nsdpoll_epollevt_lst_t *pRoot; /* Root of the epoll event list */ +}; + +/* interface is defined in nsd.h, we just implement it! */ +#define nsdpoll_ptcpCURR_IF_VERSION nsdCURR_IF_VERSION + +/* prototypes */ +PROTOTYPEObj(nsdpoll_ptcp); + +#endif /* #ifndef INCLUDED_NSDPOLL_PTCP_H */ diff --git a/runtime/nsdsel_ptcp.c b/runtime/nsdsel_ptcp.c index 41b85e0c..e2cfca7c 100644 --- a/runtime/nsdsel_ptcp.c +++ b/runtime/nsdsel_ptcp.c @@ -36,6 +36,7 @@ #include "errmsg.h" #include "nsd_ptcp.h" #include "nsdsel_ptcp.h" +#include "unlimited_select.h" /* static data */ DEFobjStaticHelpers @@ -47,14 +48,23 @@ DEFobjCurrIf(glbl) */ BEGINobjConstruct(nsdsel_ptcp) /* be sure to specify the object type also in END macro! */ pThis->maxfds = 0; +#ifdef USE_UNLIMITED_SELECT + pThis->pReadfds = calloc(1, glbl.GetFdSetSize()); + pThis->pWritefds = calloc(1, glbl.GetFdSetSize()); +#else FD_ZERO(&pThis->readfds); FD_ZERO(&pThis->writefds); +#endif ENDobjConstruct(nsdsel_ptcp) /* destructor for the nsdsel_ptcp object */ BEGINobjDestruct(nsdsel_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(nsdsel_ptcp) +#ifdef USE_UNLIMITED_SELECT + freeFdSet(pThis->pReadfds); + freeFdSet(pThis->pWritefds); +#endif ENDobjDestruct(nsdsel_ptcp) @@ -65,20 +75,27 @@ Add(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pSock, nsd_ptcp); ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); switch(waitOp) { case NSDSEL_RD: - FD_SET(pSock->sock, &pThis->readfds); + FD_SET(pSock->sock, pReadfds); break; case NSDSEL_WR: - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - FD_SET(pSock->sock, &pThis->readfds); - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pReadfds); + FD_SET(pSock->sock, pWritefds); break; } @@ -98,6 +115,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) DEFiRet; int i; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); assert(piNumReady != NULL); @@ -106,13 +130,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) // TODO: name in dbgprintf! dbgprintf("--------<NSDSEL_PTCP> calling select, active fds (max %d): ", pThis->maxfds); for(i = 0; i <= pThis->maxfds; ++i) - if(FD_ISSET(i, &pThis->readfds) || FD_ISSET(i, &pThis->writefds)) + if(FD_ISSET(i, pReadfds) || FD_ISSET(i, pWritefds)) dbgprintf("%d ", i); dbgprintf("\n"); } /* now do the select */ - *piNumReady = select(pThis->maxfds+1, &pThis->readfds, &pThis->writefds, NULL, NULL); + *piNumReady = select(pThis->maxfds+1, pReadfds, pWritefds, NULL, NULL); RETiRet; } @@ -125,6 +149,13 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); ISOBJ_TYPE_assert(pSock, nsd_ptcp); @@ -132,14 +163,14 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) switch(waitOp) { case NSDSEL_RD: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds); break; case NSDSEL_WR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds) - | FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds) + | FD_ISSET(pSock->sock, pWritefds); break; } diff --git a/runtime/nsdsel_ptcp.h b/runtime/nsdsel_ptcp.h index 6c0c7fa7..f9ec8210 100644 --- a/runtime/nsdsel_ptcp.h +++ b/runtime/nsdsel_ptcp.h @@ -31,8 +31,13 @@ typedef nsdsel_if_t nsdsel_ptcp_if_t; /* we just *implement* this interface */ struct nsdsel_ptcp_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ int maxfds; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds; + fd_set *pWritefds; +#else fd_set readfds; fd_set writefds; +#endif }; /* interface is defined in nsd.h, we just implement it! */ diff --git a/runtime/nspoll.c b/runtime/nspoll.c new file mode 100644 index 00000000..f287cd4e --- /dev/null +++ b/runtime/nspoll.c @@ -0,0 +1,198 @@ +/* nspoll.c + * + * This is an io waiter interface utilizing the much-more-efficient poll/epoll API. + * Note that it may not always be available for a given driver. If so, that is reported + * back to the upper peer which then should consult a nssel-based io waiter. + * + * Work on this module begun 2009-11-18 by Rainer Gerhards. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include "rsyslog.h" +#include "obj.h" +#include "module-template.h" +#include "netstrm.h" +#include "nspoll.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) + + +/* load our low-level driver. This must be done before any + * driver-specific functions (allmost all...) can be carried + * out. Note that the driver's .ifIsLoaded is correctly + * initialized by calloc() and we depend on that. Please note that + * we do some name-mangeling. We know that each nsd driver also needs + * a nspoll driver. So we simply append "sel" to the nsd driver name: This, + * of course, means that the driver name must match these rules, but that + * shouldn't be a real problem. + * WARNING: this code is mostly identical to similar code in + * netstrms.c - TODO: abstract it and move it to some common place. + * rgerhards, 2008-04-28 + */ +static rsRetVal +loadDrvr(nspoll_t *pThis) +{ + DEFiRet; + uchar *pBaseDrvrName; + uchar szDrvrName[48]; /* 48 shall be large enough */ + + pBaseDrvrName = pThis->pBaseDrvrName; + if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdpoll_%s", pBaseDrvrName) == sizeof(szDrvrName)) + ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); + CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); + + pThis->Drvr.ifVersion = nsdCURR_IF_VERSION; + /* The pDrvrName+2 below is a hack to obtain the object name. It + * safes us to have yet another variable with the name without "lm" in + * front of it. If we change the module load interface, we may re-think + * about this hack, but for the time being it is efficient and clean + * enough. -- rgerhards, 2008-04-18 + */ +RUNLOG_VAR("%s", szDrvrName+2); + CHKiRet(obj.UseObj(__FILE__, szDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr)); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pDrvrName != NULL) + free(pThis->pDrvrName); + pThis->pDrvrName = NULL; + } + RETiRet; +} + + +/* Standard-Constructor */ +BEGINobjConstruct(nspoll) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(nspoll) + + +/* destructor for the nspoll object */ +BEGINobjDestruct(nspoll) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nspoll) + if(pThis->pDrvrData != NULL) + pThis->Drvr.Destruct(&pThis->pDrvrData); + + /* and now we must release our driver, if we got one. We use the presence of + * a driver name string as load indicator (because we also need that string + * to release the driver + */ + if(pThis->pDrvrName != NULL) { + obj.ReleaseObj(__FILE__, pThis->pDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr); + free(pThis->pDrvrName); + } +ENDobjDestruct(nspoll) + + +/* ConstructionFinalizer */ +static rsRetVal +ConstructFinalize(nspoll_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); +RUNLOG_STR("trying to load epoll driver\n"); + CHKiRet(loadDrvr(pThis)); + CHKiRet(pThis->Drvr.Construct(&pThis->pDrvrData)); +finalize_it: +dbgprintf("XXX: done trying to load epoll driver, state %d\n", iRet); + RETiRet; +} + + +/* Carries out the actual wait (all done in lower layers) + */ +static rsRetVal +Wait(nspoll_t *pThis, int timeout, int *idRdy, void **ppUsr) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + assert(idRdy != NULL); + iRet = pThis->Drvr.Wait(pThis->pDrvrData, timeout, idRdy, ppUsr); + RETiRet; +} + + +/* semantics like the epoll_ctl() function, does the same thing. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Ctl(nspoll_t *pThis, netstrm_t *pStrm, int id, void *pUsr, int mode, int op) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + iRet = pThis->Drvr.Ctl(pThis->pDrvrData, pStrm->pDrvrData, id, pUsr, mode, op); + RETiRet; +} + + +/* queryInterface function */ +BEGINobjQueryInterface(nspoll) +CODESTARTobjQueryInterface(nspoll) + if(pIf->ifVersion != nspollCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = nspollConstruct; + pIf->ConstructFinalize = ConstructFinalize; + pIf->Destruct = nspollDestruct; + pIf->Wait = Wait; + pIf->Ctl = Ctl; +finalize_it: +ENDobjQueryInterface(nspoll) + + +/* exit our class + */ +BEGINObjClassExit(nspoll, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nspoll) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(nspoll) + + +/* Initialize the nspoll class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nspoll, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + DBGPRINTF("doing nspollClassInit\n"); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nspoll) +/* vi:set ai: + */ diff --git a/runtime/nspoll.h b/runtime/nspoll.h new file mode 100644 index 00000000..a77759c0 --- /dev/null +++ b/runtime/nspoll.h @@ -0,0 +1,65 @@ +/* Definitions for the nspoll io activity waiter + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSPOLL_H +#define INCLUDED_NSPOLL_H + +#include "netstrms.h" + +/* some operations to be portable when we do not have epoll() available */ +#define NSDPOLL_ADD 1 +#define NSDPOLL_DEL 2 + +/* and some mode specifiers for waiting on input/output */ +#define NSDPOLL_IN 1 /* EPOLLIN */ +#define NSDPOLL_OUT 2 /* EPOLLOUT */ +/* next is 4, 8, 16, ... - must be bit values, as they are ored! */ + +/* the nspoll object */ +struct nspoll_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + nsd_t *pDrvrData; /**< the driver's data elements */ + uchar *pBaseDrvrName; /**< nsd base driver name to use, or NULL if system default */ + uchar *pDrvrName; /**< full base driver name (set when driver is loaded) */ + nsdpoll_if_t Drvr; /**< our stream driver */ +}; + + +/* interface */ +BEGINinterface(nspoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nspoll_t **ppThis); + rsRetVal (*ConstructFinalize)(nspoll_t *pThis); + rsRetVal (*Destruct)(nspoll_t **ppThis); + rsRetVal (*Wait)(nspoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); + rsRetVal (*Ctl)(nspoll_t *pNsdpoll, netstrm_t *pStrm, int id, void *pUsr, int mode, int op); + rsRetVal (*IsEPollSupported)(void); /* static method */ +ENDinterface(nspoll) +#define nspollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + +/* prototypes */ +PROTOTYPEObj(nspoll); + +/* the name of our library binary */ +#define LM_NSPOLL_FILENAME LM_NETSTRMS_FILENAME + +#endif /* #ifndef INCLUDED_NSPOLL_H */ diff --git a/runtime/nssel.c b/runtime/nssel.c index d11d5fe1..7c5be3a9 100644 --- a/runtime/nssel.c +++ b/runtime/nssel.c @@ -219,6 +219,7 @@ ENDObjClassExit(nssel) */ BEGINObjClassInit(nssel, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + DBGPRINTF("doing nsselClassInit\n"); CHKiRet(objUse(glbl, CORE_COMPONENT)); /* set our own handlers */ diff --git a/runtime/parser.c b/runtime/parser.c index 38f72986..b5245795 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -60,6 +60,7 @@ DEFobjCurrIf(ruleset) /* config data */ static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ /* This is the list of all parsers known to us. @@ -339,6 +340,11 @@ SanitizeMsg(msg_t *pMsg) * needs sanitation than to do the sanitation in any case. So we first do * this and terminate when it is not needed - which is expectedly the case * for the vast majority of messages. -- rgerhards, 2009-06-15 + * Note that we do NOT check here if tab characters are to be escaped or + * not. I expect this functionality to be seldomly used and thus I do not + * like to pay the performance penalty. So the penalty is only with those + * that actually use it, because we may call the sanitizer without actual + * need below (but it then still will work perfectly well!). -- rgerhards, 2009-11-27 */ int bNeedSanitize = 0; for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { @@ -367,7 +373,7 @@ SanitizeMsg(msg_t *pMsg) CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ - if(iscntrl((int) pszMsg[iSrc])) { + if(iscntrl((int) pszMsg[iSrc]) && (pszMsg[iSrc] != '\t' || bEscapeTab)) { /* note: \0 must always be escaped, the rest of the code currently * can not handle it! -- rgerhards, 2009-08-26 */ @@ -462,7 +468,8 @@ ParseMsg(msg_t *pMsg) * (and that functionality is too important for debugging to drop it...). */ DBGPRINTF("msg parser: flags %x, from '%s', msg '%.50s'\n", pMsg->msgFlags, - getRcvFrom(pMsg), pMsg->pszRawMsg); + (pMsg->msgFlags & NEEDS_DNSRESOL) ? UCHAR_CONSTANT("~NOTRESOLVED~") : getRcvFrom(pMsg), + pMsg->pszRawMsg); /* we now need to go through our list of parsers and see which one is capable of * parsing the message. Note that the first parser that requires message sanitization @@ -618,6 +625,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus { cCCEscapeChar = '#'; bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bEscapeTab = 1; /* default is to escape control characters */ bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ return RS_RET_OK; @@ -669,6 +677,7 @@ BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); InitParserList(&pParsLstRoot); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index a5e8cf5d..1431c684 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -107,6 +107,7 @@ typedef struct NetAddr netAddr_t; typedef struct netstrms_s netstrms_t; typedef struct netstrm_s netstrm_t; typedef struct nssel_s nssel_t; +typedef struct nspoll_s nspoll_t; typedef enum nsdsel_waitOp_e nsdsel_waitOp_t; typedef struct nsd_ptcp_s nsd_ptcp_t; typedef struct nsd_gtls_s nsd_gtls_t; @@ -114,9 +115,11 @@ typedef struct nsd_gsspi_s nsd_gsspi_t; typedef struct nsd_nss_s nsd_nss_t; typedef struct nsdsel_ptcp_s nsdsel_ptcp_t; typedef struct nsdsel_gtls_s nsdsel_gtls_t; +typedef struct nsdpoll_ptcp_s nsdpoll_ptcp_t; typedef struct wti_s wti_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; +typedef obj_t nsdpoll_t; typedef struct msg msg_t; typedef struct queue_s qqueue_t; typedef struct prop_s prop_t; @@ -148,6 +151,8 @@ typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif +typedef struct epoll_event epoll_event_t; + typedef char bool; /* I intentionally use char, to keep it slim so that many fit into the CPU cache! */ /* settings for flow control @@ -409,6 +414,11 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */ RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */ RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */ + RS_RET_EINTR = -2161, /**< EINTR occured during a system call (not necessarily an error) */ + RS_RET_ERR_EPOLL = -2162, /**< epoll() returned with an unexpected error code */ + RS_RET_ERR_EPOLL_CTL = -2163, /**< epol_ctll() returned with an unexpected error code */ + RS_RET_TIMEOUT = -2164, /**< timeout occured during operation */ + RS_RET_RCV_ERR = -2165, /**< error occured during socket rcv operation */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/unlimited_select.h b/runtime/unlimited_select.h new file mode 100644 index 00000000..32dadc03 --- /dev/null +++ b/runtime/unlimited_select.h @@ -0,0 +1,45 @@ +/* unlimited_select.h + * Tweak the macros for accessing fd_set so that the select() syscall + * won't be limited to a particular number of file descriptors. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +#ifndef UNLIMITED_SELECT_H_INCLUDED + +#include <string.h> +#include <stdlib.h> +#include <sys/select.h> +#include "glbl.h" + +#ifdef USE_UNLIMITED_SELECT +# undef FD_ZERO +# define FD_ZERO(set) memset((set), 0, glbl.GetFdSetSize()); +#endif + +#ifdef USE_UNLIMITED_SELECT +void freeFdSet(fd_set *p) { + free(p); +} +#else +# define freeFdSet(x) +#endif + +#endif /* #ifndef UNLIMITED_SELECT_H_INCLUDED */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 060e6627..ab7ca4bb 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -442,7 +442,6 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) CHKiRet(wtpStartWrkr(pThis)); } } else { -dbgprintf("YYY: wtpAdviseMaxWorkers, sufficient workers, just doing adivse signal cond busy\n"); pthread_cond_signal(pThis->pcondBusy); } @@ -15,9 +15,6 @@ * callbacks before the code is run. The tcpsrv then calls back * into the specific input modules at the appropriate time. * - * NOTE: read comments in module-template.h to understand how this file - * works! - * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. @@ -68,6 +65,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "errmsg.h" #include "ruleset.h" #include "unicode-helper.h" @@ -89,6 +87,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) +DEFobjCurrIf(nspoll) DEFobjCurrIf(prop) @@ -238,11 +237,13 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) if(pThis->pSessions != NULL) { /* close all TCP connections! */ - i = TCPSessGetNxtSess(pThis, -1); - while(i != -1) { - tcps_sess.Destruct(&pThis->pSessions[i]); - /* now get next... */ - i = TCPSessGetNxtSess(pThis, i); + if(!pThis->bUsingEPoll) { + i = TCPSessGetNxtSess(pThis, -1); + while(i != -1) { + tcps_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = TCPSessGetNxtSess(pThis, i); + } } /* we are done with the session table - so get rid of it... */ @@ -438,7 +439,8 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, } *ppSess = pSess; - pThis->pSessions[iSess] = pSess; + if(!pThis->bUsingEPoll) + pThis->pSessions[iSess] = pSess; pSess = NULL; /* this is now also handed over */ finalize_it: @@ -465,11 +467,29 @@ RunCancelCleanup(void *arg) } +/* helper to close a session. Takes status of poll vs. select into consideration. + * rgerhards, 2009-11-25 + */ +static inline rsRetVal +closeSess(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) { + DEFiRet; + if(pPoll != NULL) { + CHKiRet(nspoll.Ctl(pPoll, (*ppSess)->pStrm, 0, *ppSess, NSDPOLL_IN, NSDPOLL_DEL)); + } + pThis->pOnRegularClose(*ppSess); + tcps_sess.Destruct(ppSess); +finalize_it: + RETiRet; +} + + /* process a receive request on one of the streams + * If pPoll is non-NULL, we have a netstream in epoll mode, which means we need + * to remove any descriptor we close from the epoll set. * rgerhards, 2009-07-020 */ static rsRetVal -doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) +doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) { char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ ssize_t iRcvd; @@ -478,7 +498,6 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) ISOBJ_TYPE_assert(pThis, tcpsrv); DBGPRINTF("netstream %p with new data\n", (*ppSess)->pStrm); - /* Receive message */ iRet = pThis->pRcvData(*ppSess, buf, sizeof(buf), &iRcvd); switch(iRet) { @@ -491,8 +510,7 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "Netstream session %p closed by remote peer %s.\n", (*ppSess)->pStrm, pszPeer); } - pThis->pOnRegularClose(*ppSess); - tcps_sess.Destruct(ppSess); + CHKiRet(closeSess(pThis, ppSess, pPoll)); break; case RS_RET_RETRY: /* we simply ignore retry - this is not an error, but we also have not received anything */ @@ -505,26 +523,29 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) */ errmsg.LogError(0, localRet, "Tearing down TCP Session - see " "previous messages for reason(s)\n"); - pThis->pOnErrClose(*ppSess); - tcps_sess.Destruct(ppSess); + CHKiRet(closeSess(pThis, ppSess, pPoll)); } break; default: errno = 0; errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", (*ppSess)->pStrm); - pThis->pOnErrClose(*ppSess); - tcps_sess.Destruct(ppSess); + CHKiRet(closeSess(pThis, ppSess, pPoll)); break; } + +finalize_it: RETiRet; } -/* This function is called to gather input. */ +/* This function is called to gather input. + * This variant here is only used if we need to work with a netstream driver + * that does not support epoll(). + */ #pragma GCC diagnostic ignored "-Wempty-body" -static rsRetVal -Run(tcpsrv_t *pThis) +static inline rsRetVal +RunSelect(tcpsrv_t *pThis) { DEFiRet; int nfds; @@ -532,7 +553,7 @@ Run(tcpsrv_t *pThis) int iTCPSess; int bIsReady; tcps_sess_t *pNewSess; - nssel_t *pSel; + nssel_t *pSel = NULL; ISOBJ_TYPE_assert(pThis, tcpsrv); @@ -583,7 +604,7 @@ Run(tcpsrv_t *pThis) ABORT_FINALIZE(RS_RET_FORCE_TERM); CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - doReceive(pThis, &pThis->pSessions[iTCPSess]); + doReceive(pThis, &pThis->pSessions[iTCPSess], NULL); --nfds; /* indicate we have processed one */ } iTCPSess = TCPSessGetNxtSess(pThis, iTCPSess); @@ -606,6 +627,87 @@ finalize_it: /* this is a very special case - this time only we do not exit the #pragma GCC diagnostic warning "-Wempty-body" +/* This function is called to gather input. It tries doing that via the epoll() + * interface. If the driver does not support that, it falls back to calling its + * select() equivalent. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Run(tcpsrv_t *pThis) +{ + DEFiRet; + int i; + tcps_sess_t *pNewSess; + nspoll_t *pPoll = NULL; + void *pUsr; + rsRetVal localRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerhards, 20080-04-24 + */ + if((localRet = nspoll.Construct(&pPoll)) == RS_RET_OK) { + // TODO: set driver + localRet = nspoll.ConstructFinalize(pPoll); + } + if(localRet != RS_RET_OK) { + /* fall back to select */ + dbgprintf("tcpsrv could not use epoll() interface, iRet=%d, using select()\n", localRet); + iRet = RunSelect(pThis); + FINALIZE; + } + + dbgprintf("tcpsrv uses epoll() interface, nsdpol driver found\n"); + + /* flag that we are in epoll mode */ + pThis->bUsingEPoll = TRUE; + + /* Add the TCP listen sockets to the list of sockets to monitor */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + dbgprintf("Trying to add listener %d, pUsr=%p\n", i, pThis->ppLstn); + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_ADD)); + dbgprintf("Added listener %d\n", i); + } + + while(1) { + localRet = nspoll.Wait(pPoll, -1, &i, &pUsr); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + + /* check if we need to ignore the i/o ready state. We do this if we got an invalid + * return state. Validly, this can happen for RS_RET_EINTR, for other cases it may + * not be the right thing, but what is the right thing is really hard at this point... + */ + if(localRet != RS_RET_OK) + continue; + + dbgprintf("poll returned with i %d, pUsr %p\n", i, pUsr); + + if(pUsr == pThis->ppLstn) { + DBGPRINTF("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + CHKiRet(nspoll.Ctl(pPoll, pNewSess->pStrm, 0, pNewSess, NSDPOLL_IN, NSDPOLL_ADD)); + DBGPRINTF("New session created with NSD %p.\n", pNewSess); + } else { + pNewSess = (tcps_sess_t*) pUsr; + doReceive(pThis, &pNewSess, pPoll); + } + } + + /* remove the tcp listen sockets from the epoll set */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_DEL)); + } + +finalize_it: + if(pPoll != NULL) + nspoll.Destruct(&pPoll); + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ pThis->iSessMax = TCPSESS_MAX_DEFAULT; @@ -908,7 +1010,6 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->ConstructFinalize = tcpsrvConstructFinalize; pIf->Destruct = tcpsrvDestruct; - //pIf->SessAccept = SessAccept; pIf->configureTCPListen = configureTCPListen; pIf->create_tcp_socket = create_tcp_socket; pIf->Run = Run; @@ -969,6 +1070,7 @@ BEGINObjClassInit(tcpsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(nspoll, DONT_LOAD_LIB)); CHKiRet(objUse(tcps_sess, DONT_LOAD_LIB)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); @@ -55,6 +55,7 @@ struct tcpsrv_s { ruleset_t *pRuleset; /**< ruleset to bind to */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ bool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ + bool bUsingEPoll; /**< are we in epoll mode (means we do not need to keep track of sessions!) */ int iLstnCurr; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ diff --git a/tests/Makefile.am b/tests/Makefile.am index 716207e1..1a7eba71 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -20,6 +20,8 @@ TESTS = $(TESTRUNS) cfg.sh \ queue-persist.sh \ pipeaction.sh \ execonlyonce.sh \ + dircreate_dflt.sh \ + dircreate_off.sh \ queue-persist.sh if ENABLE_OMUDPSPOOF @@ -37,6 +39,8 @@ TESTS += omod-if-array.sh \ threadingmqaq.sh \ discard.sh \ badqi.sh \ + tabescape_dflt.sh \ + tabescape_off.sh \ fieldtest.sh endif @@ -194,6 +198,16 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ execonlyonce.sh \ testsuites/execonlyonce.conf \ testsuites/execonlyonce.data \ + tabescape_dflt.sh \ + testsuites/tabescape_dflt.conf \ + testsuites/1.tabescape_dflt \ + tabescape_off.sh \ + testsuites/tabescape_off.conf \ + testsuites/1.tabescape_off \ + dircreate_dflt.sh \ + testsuites/dircreate_dflt.conf \ + dircreate_off.sh \ + testsuites/dircreate_off.conf \ DiagTalker.java \ cfg.sh diff --git a/tests/diag.sh b/tests/diag.sh index 98228b12..1f410645 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -18,14 +18,14 @@ case $1 in rm -f rsyslogd.started work-*.conf rm -f rsyslogd2.started work-*.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files - rm -rf test-spool + rm -rf test-spool test-logdir rm -f core.* vgcore.* mkdir test-spool ;; 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf rm -f rsyslogd2.started diag-common2.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files - rm -rf test-spool + rm -rf test-spool test-logdir echo ------------------------------------------------------------------------------- ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use diff --git a/tests/dircreate_dflt.sh b/tests/dircreate_dflt.sh new file mode 100755 index 00000000..71a671f3 --- /dev/null +++ b/tests/dircreate_dflt.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_dflt_dflt.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_dflt.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ ! -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile not created!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/dircreate_off.sh b/tests/dircreate_off.sh new file mode 100755 index 00000000..92fdee01 --- /dev/null +++ b/tests/dircreate_off.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_off_off.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_off.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile WAS created where not permitted to!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/tabescape_dflt.sh b/tests/tabescape_dflt.sh new file mode 100755 index 00000000..d0e13ec9 --- /dev/null +++ b/tests/tabescape_dflt.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_dflt.sh\]: test for default tab escaping +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_dflt -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_dflt -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/tabescape_off.sh b/tests/tabescape_off.sh new file mode 100755 index 00000000..71ede7c0 --- /dev/null +++ b/tests/tabescape_off.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_off.sh\]: test for tab escaping off +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_off -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_off -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/testsuites/1.tabescape_dflt b/tests/testsuites/1.tabescape_dflt new file mode 100644 index 00000000..91444bd3 --- /dev/null +++ b/tests/testsuites/1.tabescape_dflt @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT#011after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.tabescape_off b/tests/testsuites/1.tabescape_off new file mode 100644 index 00000000..6a331c35 --- /dev/null +++ b/tests/testsuites/1.tabescape_off @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/dircreate_dflt.conf b/tests/testsuites/dircreate_dflt.conf new file mode 100644 index 00000000..9b9aadb8 --- /dev/null +++ b/tests/testsuites/dircreate_dflt.conf @@ -0,0 +1,11 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/dircreate_off.conf b/tests/testsuites/dircreate_off.conf new file mode 100644 index 00000000..28ccbd8c --- /dev/null +++ b/tests/testsuites/dircreate_off.conf @@ -0,0 +1,12 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$CreateDirs off +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/tabescape_dflt.conf b/tests/testsuites/tabescape_dflt.conf new file mode 100644 index 00000000..b9d92a37 --- /dev/null +++ b/tests/testsuites/tabescape_dflt.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tabescape_off.conf b/tests/testsuites/tabescape_off.conf new file mode 100644 index 00000000..c1eca305 --- /dev/null +++ b/tests/testsuites/tabescape_off.conf @@ -0,0 +1,10 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +$EscapeControlCharacterTab off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tools/syslogd.c b/tools/syslogd.c index 21df6d6c..967401d2 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -569,7 +569,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) * permits us to process unmodified config files which otherwise contain a * supressor statement. */ - if(((Debug || NoFork) && bErrMsgToStderr) || iConfigVerify) { + if(((Debug == DEBUG_FULL || NoFork) && bErrMsgToStderr) || iConfigVerify) { if(LOG_PRI(pri) == LOG_ERR) fprintf(stderr, "rsyslogd: %s\n", msg); } @@ -586,6 +586,82 @@ finalize_it: RETiRet; } +/* check message against ACL set + * rgerhards, 2009-11-16 + */ +#if 0 +static inline rsRetVal +chkMsgAgainstACL() { + /* if we reach this point, we had a good receive and can process the packet received */ + /* check if we have a different sender than before, if so, we need to query some new values */ + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us + * syslog messages. If it isn't, we do not further + * process the message but log a warning (if we are + * configured to do this). + * rgerhards, 2005-09-26 + */ + *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", + (struct sockaddr *)&frominet, (char*)fromHostFQDN); + + if(!*pbIsPermitted) { + DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender %s discarded", + (char*)fromHost); + } + } + } + } +} +#endif + + +/* consumes a single messages - this function is primarily used to shuffle + * out some code from msgConsumer(). After this function, the message is + * (by definition!) considered committed. + * rgerhards, 2009-11-16 + */ +static inline rsRetVal +msgConsumeOne(msg_t *pMsg, prop_t **propFromHost, prop_t **propFromHostIP) { + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + int bIsPermitted; + DEFiRet; + + if((pMsg->msgFlags & NEEDS_ACLCHK_U) != 0) { + dbgprintf("msgConsumer: UDP ACL must be checked for message (hostname-based)\n"); + CHKiRet(net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP)); + bIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)pMsg->rcvFrom.pfrominet, (char*)fromHostFQDN, 1); + if(!bIsPermitted) { + DBGPRINTF("Message from '%s' discarded, not a permitted sender host\n", + fromHostFQDN); + ABORT_FINALIZE(RS_RET_ERR); + /* save some of the info we obtained */ + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), propFromHostIP)); + pMsg->msgFlags &= ~NEEDS_ACLCHK_U; + } + } + + if((pMsg->msgFlags & NEEDS_PARSING) != 0) + CHKiRet(parser.ParseMsg(pMsg)); + + ruleset.ProcessMsg(pMsg); +finalize_it: + RETiRet; +} + /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE @@ -597,26 +673,22 @@ static rsRetVal msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShutdownImmediate) { int i; - msg_t *pMsg; - rsRetVal localRet; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; DEFiRet; assert(pBatch != NULL); for(i = 0 ; i < pBatch->nElem && !*pbShutdownImmediate ; i++) { - pMsg = (msg_t*) pBatch->pElem[i].pUsrp; DBGPRINTF("msgConsumer processes msg %d/%d\n", i, pBatch->nElem); - if((pMsg->msgFlags & NEEDS_PARSING) != 0) { - localRet = parser.ParseMsg(pMsg); - if(localRet == RS_RET_OK) - ruleset.ProcessMsg(pMsg); - } else { - ruleset.ProcessMsg(pMsg); - } - /* if we reach this point, the message is considered committed (by definition!) */ + msgConsumeOne((msg_t*) pBatch->pElem[i].pUsrp, &propFromHost, &propFromHostIP); pBatch->pElem[i].state = BATCH_STATE_COMM; } + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); RETiRet; } @@ -905,9 +977,10 @@ static void doDie(int sig) static int iRetries = 0; /* debug aid */ dbgprintf(MSG1); if(Debug) + if(Debug == DEBUG_FULL) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { - if(Debug) + if(Debug == DEBUG_FULL) write(1, MSG2, sizeof(MSG2) - 1); abort(); } @@ -1086,6 +1159,9 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) iFiles, errStr, (long) maxFiles.rlim_max); ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); } +#ifdef USE_UNLIMITED_SELECT + glbl.SetFdSetSize(howmany(iFiles, __NFDBITS) * sizeof (fd_mask)); +#endif DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); finalize_it: @@ -2094,7 +2170,7 @@ static rsRetVal mainThread() * is still in its infancy (and not really done), we currently accept this issue. * rgerhards, 2009-06-29 */ - if(!(Debug || NoFork)) { + if(!(Debug == DEBUG_FULL || NoFork)) { close(1); close(2); bErrMsgToStderr = 0; @@ -2286,7 +2362,7 @@ doGlblProcessInit(void) thrdInit(); - if( !(Debug || NoFork) ) + if( !(Debug == DEBUG_FULL || NoFork) ) { DBGPRINTF("Checking pidfile.\n"); if (!check_pid(PidFile)) |
