diff options
83 files changed, 2058 insertions, 255 deletions
@@ -1,4 +1,31 @@ --------------------------------------------------------------------------- +Version 5.5.3 [DEVEL] (rgerhards), 2010-02-?? +- added capability to turn off standard LF delimiter in TCP server + via new directive "$InputTCPServerDisableLFDelimiter on" +- bugfix: comment char ('#') in literal terminated script parsing + and thus could not be used. + but tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=119 + [merged in from v3.22.2] +- imported patches from 4.6.0: + * improved testbench to contain samples for totally malformed messages + which miss parts of the message content + * bugfix: some malformed messages could lead to a missing LF inside files + or some other missing parts of the template content. + * bugfix: if a message ended immediately with a hostname, the hostname + was mistakenly interpreted as TAG, and localhost be used as hostname +--------------------------------------------------------------------------- +Version 5.5.2 [DEVEL] (rgerhards), 2010-02-05 +- applied patches that make rsyslog compile under Apple OS X. + Thanks to trey for providing these. +- replaced data type "bool" by "sbool" because this created some + portability issues. +- added $Escape8BitCharactersOnReceive directive + Thanks to David Lang for suggesting it. +- worked around an issue where omfile failed to compile on 32 bit platforms + under some circumstances (this smells like a gcc problem, but a simple + solution was available). Thanks to Kenneth Marshall for some advice. +- extended testbench +--------------------------------------------------------------------------- Version 5.3.8 [BETA] (rgerhards), 2010-02-?? - bugfix: omruleset failed to work in many cases bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=179 @@ -14,6 +41,45 @@ Version 5.3.7 [BETA] (rgerhards), 2010-01-27 a stack-based pointer which lead to destruction of the stack frame and thus a segfault on function return. Thanks to Michael Biebl for alerting us on this problem. +- bugfix: hostname accidently set to IP address for some message sources, + for example imudp. Thanks to Anton for reporting this bug. [imported v4] +- bugfix: ompgsql had problems with transaction support, what actually + rendered it unsuable. Thanks to forum user "horhe" for alerting me + on this bug and helping to debug/fix it! [imported from 5.3.6] +- bugfix: $CreateDirs variable not properly initialized, default thus + was random (but most often "on") [imported from v3] +--------------------------------------------------------------------------- +Version 5.5.1 [DEVEL] (rgerhards), 2009-11-27 +- introduced the ablity for netstream drivers to utilize an epoll interface + This offers increased performance and removes the select() FDSET size + limit from imtcp. Note that we fall back to select() if there is no + epoll netstream drivers. So far, an epoll driver has only been + implemented for plain tcp syslog, the rest will follow once the code + proves well in practice AND there is demand. +- re-implemented $EscapeControlCharacterTab config directive + Based on Jonathan Bond-Caron's patch for v4. This now also includes some + automatted tests. +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +--------------------------------------------------------------------------- +Version 5.5.0 [DEVEL] (rgerhards), 2009-11-18 +- moved DNS resolution code out of imudp and into the backend processing + Most importantly, DNS resolution now never happens if the resolved name + is not required. Note that this applies to imudp - for the other inputs, + DNS resolution almost comes for free, so we do not do it there. However, + the new method has been implemented in a generic way and as such may + also be used by other modules in the future. +- added option to use unlimited-size select() calls + Thanks to varmjofekoj for the patch + This is not done in imudp, as it natively supports epoll(). +- doc: improved description of what loadable modules can do +--------------------------------------------------------------------------- +Version 5.3.7 [BETA] (rgerhards), 2010-01-?? +- bugfix: potential segfaults during queue shutdown + (bugs require certain non-standard settings to appear) + Thanks to varmojfekoj for the patch [imported from 4.5.8] [backport from 5.5.2] - bugfix: wrong memory assignment for a config variable (probably without causing any harm) [backport from 5.2.2] @@ -328,7 +394,19 @@ Version 4.7.0 [v4-devel] (rgerhards), 2009-09-?? - added new config directive $omfileForceChown to (try to) fix some broken system configs. See ticket for details: http://bugzilla.adiscon.com/show_bug.cgi?id=150 +- added $EscapeControlCharacterTab config directive + Thanks to Jonathan Bond-Caron for the patch. +- added option to use unlimited-size select() calls + Thanks to varmjofekoj for the patch +- debugondemand mode caused backgrounding to fail - close to a bug, but I'd + consider the ability to background in this mode a new feature... +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +- imported changes from 4.5.7 and below +- bugfix: potential segfault when -p command line option was used + Thanks for varmojfekoj for pointing me at this bug. - imported changes from 4.5.6 and below +--------------------------------------------------------------------------- Version 4.6.1 [v4-stable] (rgerhards), 2010-02-?? - re-enabled old pipe output (using new module ompipe, built-in) after some problems with pipes (and especially in regard to xconsole) were diff --git a/Makefile.am b/Makefile.am index 5f9d35fe..52a716fd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -99,6 +99,10 @@ if ENABLE_OMRULESET SUBDIRS += plugins/omruleset endif +if ENABLE_OMDBALERTING +SUBDIRS += plugins/omdbalerting +endif + if ENABLE_OMUDPSPOOF SUBDIRS += plugins/omudpspoof endif @@ -930,6 +930,7 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem, int *pbShutdownImmedi } else { if(nElem == 1) { pBatch->pElem[pBatch->iDoneUpTo++].state = BATCH_STATE_BAD; +// TODO: This is a mark, remove when no longer needed - Here was the bug, postincrement needs to be used, not preinc bDone = 1; } else { /* retry with half as much. Depth is log_2 batchsize, so recursion is not too deep */ @@ -52,8 +52,8 @@ struct action_s { time_t tActNow; /* the current time for an action execution. Initially set to -1 and populated on an as-needed basis. This is a performance optimization. */ time_t tLastExec; /* time this action was last executed */ - bool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ - bool bWriteAllMarkMsgs;/* should all mark msgs be written (not matter how recent the action was executed)? */ + sbool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ + sbool bWriteAllMarkMsgs;/* should all mark msgs be written (not matter how recent the action was executed)? */ int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ action_state_t eState; /* current state of action */ int bHadAutoCommit; /* did an auto-commit happen during doAction()? */ @@ -67,7 +67,7 @@ struct action_s { time_t tLastOccur; /* time last occurence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ - bool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ + sbool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ diff --git a/configure.ac b/configure.ac index b297b284..24e96965 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.3.7],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.5.2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -347,6 +347,21 @@ AC_ARG_ENABLE([fsstnd], ]) +# support for unlimited select() syscall +AC_ARG_ENABLE(unlimited_select, + [AS_HELP_STRING([--enable-unlimited-select],[Enable unlimited select() syscall @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_unlimited_select="yes" ;; + no) enable_unlimited_select="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-unlimited-select) ;; + esac], + [enable_unlimited_select="no"] +) +if test "$enable_unlimited_select" = "yes"; then + AC_DEFINE(USE_UNLIMITED_SELECT, 1, [If defined, the select() syscall won't be limited to a particular number of file descriptors.]) +fi + + # debug AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], @@ -806,6 +821,20 @@ AC_ARG_ENABLE(omruleset, AM_CONDITIONAL(ENABLE_OMRULESET, test x$enable_omruleset = xyes) +# settings for omdbalerting +AC_ARG_ENABLE(omdbalerting, + [AS_HELP_STRING([--enable-omdbalerting],[Compiles omdbalerting module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omdbalerting="yes" ;; + no) enable_omdbalerting="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omdbalerting) ;; + esac], + [enable_omdbalerting=no] +) +AM_CONDITIONAL(ENABLE_OMDBALERTING, test x$enable_omdbalerting = xyes) + + + # building the GUI (mostly for diagnostic reasons) AC_ARG_ENABLE(gui, [AS_HELP_STRING([--enable-gui],[Enable GUI programs @<:@default=no@:>@])], @@ -897,6 +926,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/omruleset/Makefile \ + plugins/omdbalerting/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ @@ -926,6 +956,7 @@ echo " rsyslog runtime will be built: $enable_rsyslogrt" echo " rsyslogd will be built: $enable_rsyslogd" echo " GUI components will be built: $enable_gui" echo " custom module 1 will be built: $enable_cust1" +echo " Unlimited select() support enabled: $enable_unlimited_select" echo echo "---{ input plugins }---" echo " Klog functionality enabled: $enable_klog ($os_type)" @@ -938,6 +969,7 @@ echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" echo " omruleset module will be compiled: $enable_omruleset" +echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" echo " output template module will be compiled: $enable_omtemplate" echo diff --git a/doc/Makefile.am b/doc/Makefile.am index f1c3f871..42a0e83c 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -66,6 +66,7 @@ html_files = \ rsconf1_actionresumeinterval.html \ rsconf1_allowedsender.html \ rsconf1_controlcharacterescapeprefix.html \ + rsconf1_escape8bitcharsonreceive.html \ rsconf1_debugprintcfsyslinehandlerlist.html \ rsconf1_debugprintmodulelist.html \ rsconf1_debugprinttemplatelist.html \ @@ -123,6 +124,7 @@ html_files = \ grfx_files = \ rsyslog_confgraph_complex.png\ rsyslog_confgraph_std.png \ + module_workflow.png \ direct_queue0.png \ direct_queue1.png \ direct_queue2.png \ diff --git a/doc/build_from_repo.html b/doc/build_from_repo.html index 8d3b20fe..a06863e1 100644 --- a/doc/build_from_repo.html +++ b/doc/build_from_repo.html @@ -43,12 +43,37 @@ you downloaded an official distribution tarball (see the <a href="install.html">rsyslog install guide</a>, starting at step 2, for further details about that). +<h2>Special Compile-Time Options</h2> +<p>On some platforms, compile-time issues occur, like the one shown below: +<p><pre><code> +make[2]: Entering directory `/home/az/RSyslog/rsyslog-5.5.0/tools' + CCLD rsyslogd +rsyslogd-omfile.o: In function `getClockFileAccess': +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +</code></pre> +<p>Note that the exact error messages can be different. These type of errors stem down to +atomic instruction support in GCC, which is somewhat depending on the machine architecture it +compiles code for. Very old machines (like the original i386) do not even at all provide support +for these instructions. +<p>The availability of atomic instructions is vital for rsyslog - it can not be built without them. +Consequently, there is a configure check included for them. But under some circumstances, +GCC seems to report they are available, but does not provide implementations for +all of them (at least this is my observation...). The simple cure is to make sure that +GCC generates code for a modern-enough architecture. This, for example, can be done as +follows: +<p><pre><code> +./configure CFLAGS="-march=i586 -mcpu=i686" --enable-imfile ... (whatever you need) +</code></pre> +<p>These settings should resolve the issue . + <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 1.2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/debug.html b/doc/debug.html index 46759986..6aeb7975 100644 --- a/doc/debug.html +++ b/doc/debug.html @@ -138,7 +138,7 @@ instance of rsyslogd can be aborted by pressing ctl-c. <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body> diff --git a/doc/imtcp.html b/doc/imtcp.html index 434b3903..422bbd55 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -23,7 +23,7 @@ versions do NOT support it. </p> <p><b>Configuration Directives</b>:</p> <ul> -<li>$InputTCPServerAddtlFrameDelimiter <Delimiter><br> +<li><b>$InputTCPServerAddtlFrameDelimiter <Delimiter></b><br> This directive permits to specify an additional frame delimiter for plain tcp syslog. The industry-standard specifies using the LF character as frame delimiter. Some vendors, notable Juniper in their NetScreen products, use an invalid frame delimiter, in Juniper's @@ -43,28 +43,36 @@ very limited interest in fixing this issue. This directive <b>can not</b> fix th That would require much more code changes, which I was unable to do so far. Full details can be found at the <a href="http://www.rsyslog.com/Article321.phtml">Cisco tcp syslog anomaly</a> page. -<li>$InputTCPServerNotifyOnConnectionClose [on/<b>off</b>] (available since 4.5.5)<br> +<li><b>$InputTCPServerDisableLFDelimiter</b> <on/<b>off</b>> (available since 5.5.3)<br> +Industry-strandard plain text tcp syslog uses the LF to delimit syslog frames. However, +some users brought up the case that it may be useful to define a different delimiter and +totally disable LF as a delimiter (the use case named were multi-line messages). This mode +is non-standard and will probably come with a lot of problems. However, as there is need +for it and it is relatively easy to support, we do so. Be sure to turn this setting to +"on" only if you exactly know what you are doing. You may run into all sorts of troubles, +so be prepared to wrangle with that! +<li><b>$InputTCPServerNotifyOnConnectionClose</b> [on/<b>off</b>] (available since 4.5.5)<br> instructs imtcp to emit a message if the remote peer closes a connection.<br> <b>Important:</b> This directive is global to all listeners and must be given right after loading imtcp, otherwise it may have no effect.</li> -<li>$InputTCPServerRun <port><br> +<li><b>$InputTCPServerRun</b> <port><br> Starts a TCP server on selected port</li> -<li>$InputTCPMaxListeners <number><br> +<li><b>$InputTCPMaxListeners</b> <number><br> Sets the maximum number of listeners (server ports) supported. Default is 20. This must be set before the first $InputTCPServerRun directive.</li> -<li>$InputTCPMaxSessions <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> -<li>$InputTCPServerStreamDriverMode <number><br> +<li><b>$InputTCPMaxSessions</b> <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> +<li><b>$InputTCPServerStreamDriverMode</b> <number><br> Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specifc.</li> -<li>$InputTCPServerInputName <name><br> +<li><b>$InputTCPServerInputName</b> <name><br> Sets a name for the inputname property. If no name is set "imtcp" is used by default. Setting a name is not strictly necessary, but can be useful to apply filtering based on which input the message was received from. -<li>$InputTCPServerStreamDriverAuthMode <mode-string><br> +<li><b>$InputTCPServerStreamDriverAuthMode</b> <mode-string><br> Sets the authentication mode for the currently selected <a href="netstream.html">network stream driver</a>. <mode-string> is driver specifc.</li> -<li>$InputTCPServerStreamDriverPermittedPeer <id-string><br> +<li><b>$InputTCPServerStreamDriverPermittedPeer</b> <id-string><br> Sets permitted peer IDs. Only these peers are able to connect to the listener. <id-string> semantics depend on the currently selected AuthMode and <a href="netstream.html">network stream driver</a>. PermittedPeers may not be set in anonymous modes.</li> -<li>$InputTCPServerBindRuleset <ruleset><br> +<li><b>$InputTCPServerBindRuleset</b> <ruleset><br> Binds the listener to a specific <a href="multi_ruleset.html">ruleset</a>.</li> </ul> <b>Caveats/Known Bugs:</b> diff --git a/doc/manual.html b/doc/manual.html index ddfcb253..bcdd64cf 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 5.3.7 (beta branch) of rsyslog.</b> +<p><b>This documentation is for version 5.5.2 (devel branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might @@ -71,6 +71,7 @@ syslog sender over NAT</a> (online only)</li> <li><a href="debug.html">debug support in rsyslog</a></li> <li>Developer Documentation <ul> + <li><a href="build_from_repo.html">building rsyslog from the source repository</a></li> <li><a href="dev_oplugins.html">writing rsyslog output plugins</a></li> <li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> </ul></li> diff --git a/doc/module_workflow.png b/doc/module_workflow.png Binary files differnew file mode 100644 index 00000000..e1a72e96 --- /dev/null +++ b/doc/module_workflow.png diff --git a/doc/rsconf1_escape8bitcharsonreceive.html b/doc/rsconf1_escape8bitcharsonreceive.html new file mode 100644 index 00000000..408851c1 --- /dev/null +++ b/doc/rsconf1_escape8bitcharsonreceive.html @@ -0,0 +1,44 @@ +<html> +<head> +<title>rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">back</a> + +<h2>$Escape8BitCharactersOnReceive</h2> +<p><b>Type:</b> global configuration directive</p> +<p><b>Default:</b> off</p> +<p><b>Available Since:</b> 5.5.2</p> +<p><b>Description:</b></p> +<p>This directive instructs rsyslogd to replace non US-ASCII characters (those that +have the 8th bit set) during reception of the message. +This may be useful for some systems. +Please note that this escaping breaks Unicode and many other encodings. Most importantly, +it can be assumed that Asian and European characters will be rendered hardly readable by +this settings. However, it may still be useful when the logs themself are primarily +in English and only occasionally contain local script. +If this option is turned on, all control-characters are converted to a 3-digit octal number and be prefixed with the $ControlCharacterEscapePrefix character (being '#' by default). +<p><b>Warning:</b></p> +<ul> + <li>turning on this option most probably destroys non-western character sets + (like Japanese, Chinese and Korean) as well as European character sets.</li> + <li>turning on this option destroys digital signatures if such exists inside + the message</li> + <li>if turned on, the drop-cc, space-cc and escape-cc + <a href="property_replacer.html">property replacer</a> options do not work + as expected because control characters are already removed upon message + reception. If you intend to use these property replacer options, you must + turn off $Escape8BitCharactersOnReceive.</li> +</ul> +<p><b>Sample:</b></p> +<p><code><b>$Escape8BitCharactersOnReceive on</b></code></p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index beb90e02..3e926db1 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -140,7 +140,9 @@ our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a <li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li> <li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li> <li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li> +<li><a href="rsconf1_escape8bitcharsonreceive.html">$Escape8BitCharactersOnReceive</a></li> <li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> +<li><b>$EscapeControlCharactersOnReceive</b> [<b>on</b>|off] - escape USASCII HT character</li> <li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> <li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li> <li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li> diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 4ce62b38..a246d0ca 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -19,8 +19,16 @@ modules solve your need, you may consider writing one or have one written for you by <a href="http://www.rsyslog.com/professional-services">Adiscon's professional services for rsyslog</a> </b>(this often is a very cost-effective and efficient way of getting what you need). +<p>There exist different classes of loadable modules: +<ul> +<li><a href="rsyslog_conf_modules.html#im">Input Modules</a> +<li><a href="rsyslog_conf_modules.html#om">Output Modules</a> +<li><a href="rsyslog_conf_modules.html#pm">Parser Modules</a> +<li><a href="rsyslog_conf_modules.html#mm">Message Modification Modules</a> +<li><a href="rsyslog_conf_modules.html#lm">Library Modules</a> +</ul> -<h2>Input Modules</h2> +<a name"im"></a><h2>Input Modules</h2> <p>Input modules are used to gather messages from various sources. They interface to message generators. <ul> @@ -35,7 +43,7 @@ to message generators. <li><a href="im3195.html">im3195</a> - accepts syslog messages via RFC 3195</li> </ul> -<h2>Output Modules</h2> +<a name"om"></a><h2>Output Modules</h2> <p>Output modules process messages. With them, message formats can be transformed and messages be transmitted to various different targets. <ul> @@ -52,14 +60,54 @@ SQLLite, Ingres, Oracle, mSQL)</li> <li><a href="ommail.html">ommail</a> - permits rsyslog to alert folks by mail if something important happens</li> <li><a href="omoracle.html">omoracle</a> - output module for Oracle (native OCI interface)</li> +<li><a href="omudpspoof.html">omudpspoof</a> - output module sending UDP syslog messages with a spoofed address</li> +</ul> + +<a name="pm"></a><h2>Parser Modules</h2> +<p>Parser modules are used to parse message content, once the message has been +received. They can be used to process custom message formats or invalidly formatted +messages. For details, please see the <a href="messageparser.html">rsyslog +message parser documentation</a>. +<p>The current modules are currently provided as part of rsyslog: +<ul> +<li>pmrfc5424 - parses RFC5424-formatted messages (the new syslog standard) +<li>pmrfc3164 - the traditional/legacy syslog parser </ul> -<h2>Library Modules</h2> +<a name="mm"></a><h2>Message Modification Modules</h2> +<p>Message modification modules are used to change the content of messages being processed. +They can be implemented using either the output module or the parser module interface. +From the rsyslog core's point of view, they actually are output or parser modules, it is their +implementation that makes them special. +<p>Currently, there do not exist any such modules, but could be written with +the methods the engine provides. They could be used, for example, to: +<ul> +<li>anonymize message content +<li>add dynamically computed content to message (fields) +</ul> + +<a name="lm"></a><h2>Library Modules</h2> <p>Library modules provide dynamically loadable functionality for parts of rsyslog, most often for other loadable modules. They can not be user-configured and are loaded automatically by some components. They are just mentioned so that error messages that point to library moduls can be understood. No module list is provided. +<h2>Where are the modules integrated into the Message Flow?</h2> +<p>Depending on their module type, modules may access and/or modify messages at +various stages during rsyslog's processing. Note that only the "core type" (e.g. input, +output) but not any type derived from it (message modification module) specifies when +a module is called. +<p>The simplified workflow is as follows: +<p align="center"> +<img src="module_workflow.png" alt"rsyslog: loadable modules and message flow"> +<p>As can be seen, messages are received by input modules, then passed to one or many +parser modules, which generate the in-memory representation of the message and may +also modify the message itself. The, the internal representation is passed to +output modules, which may output a message and (with the interfaces newly introduced +in v5) may also modify messageo object content. +<p>Note that the actual flow is much more complex and depends a lot on queue and +filter settings. This graphic above is a high-level message flow diagram. + <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> diff --git a/doc/src/module_workflow.dia b/doc/src/module_workflow.dia Binary files differnew file mode 100644 index 00000000..178571f4 --- /dev/null +++ b/doc/src/module_workflow.dia diff --git a/doc/src/tls.dia b/doc/src/tls.dia Binary files differindex 77e5d185..d7c9811d 100644 --- a/doc/src/tls.dia +++ b/doc/src/tls.dia diff --git a/doc/status.html b/doc/status.html index ff056489..801decfc 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,37 +2,30 @@ <html><head><title>rsyslog status page</title></head> <body> <h2>rsyslog status page</h2> -<p>This page reflects the status as of 2009-11-05.</p> +<p>This page reflects the status as of 2010-02-10.</p> <h2>Current Releases</h2> -<p><b>v5 development:</b> 5.3.4 [2009-11-04] - -<a href="http://www.rsyslog.com/Article423.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-185.phtml">download</a> -<br> +<p><b>development:</b> 5.5.2 [2010-02-05] - +<a href="http://www.rsyslog.com/Article439.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-193.phtml">download</a> <!-- not at the moment! +<br> <b>v4 development:</b> 4.5.1 [2009-07-15] - <a href="http://www.rsyslog.com/Article388.phtml">change log</a> - <a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-167.phtml">download</a></p> --> -<!-- not at the moment! -<br><b>v5-beta:</b> 5.1.6 [2009-10-15] - -<a href="http://www.rsyslog.com/Article413.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-180.phtml">download</a> ---> - -<br><b>v4-beta:</b> 4.5.6 [2009-11-05] - -<a href="http://www.rsyslog.com/Article425.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-186.phtml">download</a></p> +<p><b>beta:</b> 5.3.7 [2010-01-27] - +<a href="http://www.rsyslog.com/Article437.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-192.phtml">download</a></p> -<p><b>v5 stable:</b> 5.2.0 [2009-11-02] (recommended to use -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-183.phtml">5.3.3</a> instead) - +<p><b>v5 stable:</b> 5.2.0 [2009-11-02] (recommended to use v5-beta instead) - <a href="http://www.rsyslog.com/Article421.phtml">change log</a> - <a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-184.phtml">download</a> -<br><b>v4 stable:</b> 4.4.2 [2009-10-09] - -<a href="http://www.rsyslog.com/Article409.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-179.phtml">download</a> +<br><b>v4 stable:</b> 4.6.0 [2010-02-24] - +<a href="http://www.rsyslog.com/Article444.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-195.phtml">download</a> <br><b>v3 stable:</b> 3.22.1 [2009-07-02] - <a href="http://www.rsyslog.com/Article381.phtml">change log</a> - diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index a8855fd4..16b2754b 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -102,13 +102,63 @@ comes without any guarantees, include no guarantee on confidentiality [aka "we don't want to be sued for work were are not even paid for ;)]. <b>So if you submit debug logs, do so at your sole risk</b>. By submitting them, you accept this policy. +<p><b>Segmentation Faults</b> +<p>Rsyslog has a very rapid development process, complex capabilities and now gradually gets +more and more exposure. While we are happy about this, it also has some bad effects: some +deployment scenarios have probably never been tested and it may be impossible to test +them for the development team because of resources needed. So while we try to avoid this, +you may see a serious problem during deployments in demanding, non-standard, environments +(hopefully not with a stable version, but chances are good you'll run into troubles with +the development versions). +<p>Active support from the user base is very important to help us track down those things. +Most often, serious problems are the result of some memory misadressing. During development, +we routinely use valgrind, a very well and capable memory debugger. This helps us to create +pretty clean code. But valgrind can not detect anything, most importantly not code pathes +that are never executed. So of most use for us is information about aborts and abort locations. +<p>Unforutnately, faults rooted in adressing errors typically show up only later, so the +actual abort location is in an unrelated spot. To help track down the original spot, +<a href="http://www.gnu.org/software/hello/manual/libc/Heap-Consistency-Checking.html">libc +later than 5.4.23 offers support</a> for finding, and possible temporary relief from it, +by means of the MALLOC_CHECK_ environment variable. Setting it to 2 is a useful troubleshooting +aid for us. It will make the program abort as soon as the check routines detect anything +suspicious (unfortunately, this may still not be the root cause, but hopefully closer to it). +Setting it to 0 may even make some problems disappear (but it will NOT fix them!). +With functionality comes cost, and so exporting MALLOC_CHECK_ without need comes at +a performance penalty. However, we strongly recommend adding this instrumentation to your +test environment should you see any serious problems. Chances are good it will help us +interpret a dump better, and thus be able to quicker craft a fix. +<p>In order to get useful information, we need some backtrace of the abort. First, you need +to make sure that a core file is created. Under Fedora, for example, that means you need +to have an "ulimit -c unlimited" in place. +<p>Now let's assume you got a core file (e.g. in /core.1234). So what to do next? Sending a +core file to us is most often pointless - we need to have the exact same system configuration in +order to interpret it correctly. Obviously, chances are extremely slim for this to be. So we would +appreciate if you could extract the most important information. This is done as follows: +<ul> +<li>$gdb /path/to/rsyslogd +<li>$info thread +<li>you'll see a number of threads (in the range 0 to n with n being the highest number). For + <b>each</b> of them, do the following (let's assume that i is the thread number): + <ul> + <li>$ thread i (e.g. thread 0, thread 1, ...) + <li>$bt + </ul> +<li>then you can quit gdb with "$q" +</ul> +<p>Then please send all information that gdb spit out to the development team. It is best to first +ask on the forum or mailing list on how to do that. The developers will keep in contact with you +and, I fear, will probably ask for other things as well ;) +<p>Note that we strive for highest reliability of the engine even in unusual deployment scenarios. +Unfortunately, this is hard to achieve, especially with limited resources. So we are depending on +cooperation from users. This is your chance to make a big contribution to the project without the +need to program or do anything else except get a problem solved ;) <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> @@ -52,11 +52,14 @@ #include "errmsg.h" #include "gss-misc.h" #include "debug.h" +#include "glbl.h" +#include "unlimited_select.h" MODULE_TYPE_LIB /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) static void display_status_(char *m, OM_uint32 code, int type) @@ -109,28 +112,38 @@ static int read_all(int fd, char *buf, unsigned int nbyte) { int ret; char *ptr; - fd_set rfds; struct timeval tv; +#ifdef USE_UNLIMITED_SELECT + fd_set *pRfds = malloc(glbl.GetFdSetSize()); +#else + fd_set rfds; + fd_set *pRfds = &rfds; +#endif for (ptr = buf; nbyte; ptr += ret, nbyte -= ret) { - FD_ZERO(&rfds); - FD_SET(fd, &rfds); + FD_ZERO(pRfds); + FD_SET(fd, pRfds); tv.tv_sec = 1; tv.tv_usec = 0; - if ((ret = select(FD_SETSIZE, &rfds, NULL, NULL, &tv)) <= 0 - || !FD_ISSET(fd, &rfds)) + if ((ret = select(FD_SETSIZE, pRfds, NULL, NULL, &tv)) <= 0 + || !FD_ISSET(fd, pRfds)) { + freeFdSet(pRfds); return ret; + } ret = recv(fd, ptr, nbyte, 0); if (ret < 0) { if (errno == EINTR) continue; + freeFdSet(pRfds); return (ret); } else if (ret == 0) { + freeFdSet(pRfds); return (ptr - buf); } } + freeFdSet(pRfds); return (ptr - buf); } @@ -265,6 +278,7 @@ BEGINObjClassExit(gssutil, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END M CODESTARTObjClassExit(gssutil) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); ENDObjClassExit(gssutil) @@ -275,6 +289,7 @@ ENDObjClassExit(gssutil) BEGINAbstractObjClassInit(gssutil, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE class also in END MACRO! */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(gssutil) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 2f7e5fee..81b357ef 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -270,6 +270,11 @@ waitMainQEmpty(tcps_sess_t *pSess) dbgprintf("imdiag sleeping, wait mainq drain, curr size %d\n", iMsgQueueSize); srSleep(0,2); /* wait a little bit */ CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + if(iMsgQueueSize == 0) { + /* verify that queue is still empty (else it could just be a race!) */ + srSleep(1,5); /* wait a little bit */ + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + } } CHKiRet(sendResponse(pSess, "mainqueue empty\n")); diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index 1d4e3b4f..dd3d67e3 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -58,6 +58,7 @@ #include "netstrm.h" #include "glbl.h" #include "debug.h" +#include "unlimited_select.h" MODULE_TYPE_INPUT @@ -178,10 +179,10 @@ isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void *pUsrSrv, void*p pGSess = (gss_sess_t*) pUsrSess; if((pGSrv->allowedMethods & ALLOWEDMETHOD_TCP) && - net.isAllowedSender((uchar*)"TCP", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"TCP", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_TCP; if((pGSrv->allowedMethods & ALLOWEDMETHOD_GSS) && - net.isAllowedSender((uchar*)"GSS", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"GSS", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_GSS; if(allowedMethods && pGSess != NULL) pGSess->allowedMethods = allowedMethods; @@ -417,15 +418,20 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) CHKiRet(netstrm.GetSock(pSess->pStrm, &fdSess)); // TODO: method access! if (allowedMethods & ALLOWEDMETHOD_TCP) { int len; - fd_set fds; struct timeval tv; +#ifdef USE_UNLIMITED_SELECT + fd_set *pFds = malloc(glbl.GetFdSetSize()); +#else + fd_set fds; + fd_set *pFds = &fds; +#endif do { - FD_ZERO(&fds); - FD_SET(fdSess, &fds); + FD_ZERO(pFds); + FD_SET(fdSess, pFds); tv.tv_sec = 1; tv.tv_usec = 0; - ret = select(fdSess + 1, &fds, NULL, NULL, &tv); + ret = select(fdSess + 1, pFds, NULL, NULL, &tv); } while (ret < 0 && errno == EINTR); if (ret < 0) { errmsg.LogError(0, RS_RET_ERR, "TCP session %p will be closed, error ignored\n", pSess); @@ -478,6 +484,8 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) pGSess->allowedMethods = ALLOWEDMETHOD_TCP; ABORT_FINALIZE(RS_RET_OK); // TODO: define good error codes } + + freeFdSet(pFds); } context = &pGSess->gss_context; diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 176b5b18..0cfae057 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -86,6 +86,7 @@ static int iTCPLstnMax = 20; /* max number of sessions */ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ static int bEmitMsgOnClose = 0; /* emit an informational message on close by remote peer */ static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ +static int bDisableLFDelim = 0; /* disbale standard LF delimiter */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ @@ -97,7 +98,7 @@ static int isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) { - return net.isAllowedSender(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN); + return net.isAllowedSender2(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN, 1); } @@ -198,6 +199,7 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); + CHKiRet(tcpsrv.SetbDisableLFDelim(pOurTcpsrv, bDisableLFDelim)); CHKiRet(tcpsrv.SetNotificationOnRemoteClose(pOurTcpsrv, bEmitMsgOnClose)); /* now set optional params, but only if they were actually configured */ if(pszStrmDrvrAuthMode != NULL) { @@ -288,6 +290,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus iStrmDrvrMode = 0; bEmitMsgOnClose = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + bDisableLFDelim = 0; free(pszInputName); pszInputName = NULL; free(pszStrmDrvrAuthMode); @@ -334,6 +337,8 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserveraddtlframedelimiter"), 0, eCmdHdlrInt, NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverdisablelfdelimiter"), 0, eCmdHdlrBinary, + NULL, &bDisableLFDelim, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverinputname"), 0, eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverbindruleset"), 0, diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 307b684f..07a07d74 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -63,6 +63,7 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(prop) DEFobjCurrIf(ruleset) +static int bDoACLCheck; /* are ACL checks neeed? Cached once immediately before listener startup */ static int iMaxLine; /* maximum UDP message size supported */ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitted sender was last discarded * This shall prevent remote DoS when the "discard on disallowed sender" @@ -117,7 +118,6 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) if(udpLstnSocks == NULL) { /* esay, we can just replace it */ udpLstnSocks = newSocks; -RUNLOG_VAR("%d", newSocks[0]); CHKmalloc(udpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (newSocks[0] + 1))); for(iDst = 1 ; iDst <= newSocks[0] ; ++iDst) udpRulesets[iDst] = pBindRuleset; @@ -200,7 +200,7 @@ finalize_it: */ static inline rsRetVal processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, - uchar *fromHost, uchar *fromHostFQDN, uchar *fromHostIP, ruleset_t *pRuleset) + ruleset_t *pRuleset) { DEFiRet; int iNbrTimeUsed; @@ -235,37 +235,39 @@ processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, /* if we reach this point, we had a good receive and can process the packet received */ /* check if we have a different sender than before, if so, we need to query some new values */ - if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { - CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); - memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ - /* Here we check if a host is permitted to send us - * syslog messages. If it isn't, we do not further - * process the message but log a warning (if we are - * configured to do this). - * rgerhards, 2005-09-26 - */ - *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", - (struct sockaddr *)&frominet, (char*)fromHostFQDN); - - if(!*pbIsPermitted) { - DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); - if(glbl.GetOption_DisallowWarning) { - time_t tt; - - datetime.GetTime(&tt); - if(tt > ttLastDiscard + 60) { - ttLastDiscard = tt; - errmsg.LogError(0, NO_ERRCODE, - "UDP message from disallowed sender %s discarded", - (char*)fromHost); + if(bDoACLCheck) { + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us syslog messages. If it isn't, + * we do not further process the message but log a warning (if we are + * configured to do this). However, if the check would require name resolution, + * it is postponed to the main queue. See also my blog post at + * http://blog.gerhards.net/2009/11/acls-imudp-and-accepting-messages.html + * rgerhards, 2009-11-16 + */ + *pbIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)&frominet, "", 0); + + if(*pbIsPermitted == 0) { + DBGPRINTF("msg is not from an allowed sender\n"); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender discarded"); + } } } } + } else { + *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d)/%s,acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, fromHost, *pbIsPermitted, pRcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); - if(*pbIsPermitted) { + if(*pbIsPermitted != 0) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); } @@ -275,9 +277,10 @@ processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, MsgSetInputName(pMsg, pInputName); MsgSetRuleset(pMsg, pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); - CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; + if(*pbIsPermitted == 2) + pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */ + CHKiRet(msgSetFromSockinfo(pMsg, &frominet)); CHKiRet(submitMsg(pMsg)); } } @@ -307,9 +310,6 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) int i; struct sockaddr_storage frominetPrev; int bIsPermitted; - uchar fromHost[NI_MAXHOST]; - uchar fromHostIP[NI_MAXHOST]; - uchar fromHostFQDN[NI_MAXHOST]; struct epoll_event *udpEPollEvt = NULL; struct epoll_event currEvt[NUM_EPOLL_EVENTS]; char errStr[1024]; @@ -359,7 +359,7 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) for(i = 0 ; i < nfds ; ++i) { processSocket(pThrd, udpLstnSocks[currEvt[i].data.u64], &frominetPrev, &bIsPermitted, - fromHost, fromHostFQDN, fromHostIP, udpRulesets[currEvt[i].data.u64]); + udpRulesets[currEvt[i].data.u64]); } } @@ -377,7 +377,6 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) int maxfds; int nfds; int i; - fd_set readfds; struct sockaddr_storage frominetPrev; int bIsPermitted; uchar fromHost[NI_MAXHOST]; @@ -399,21 +398,21 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) * is given without -a, we do not need to listen at all.. */ maxfds = 0; - FD_ZERO (&readfds); + FD_ZERO (pReadfds); /* Add the UDP listen sockets to the list of read descriptors. */ for (i = 0; i < *udpLstnSocks; i++) { if (udpLstnSocks[i+1] != -1) { if(Debug) net.debugListenInfo(udpLstnSocks[i+1], "UDP"); - FD_SET(udpLstnSocks[i+1], &readfds); + FD_SET(udpLstnSocks[i+1], pReadfds); if(udpLstnSocks[i+1]>maxfds) maxfds=udpLstnSocks[i+1]; } } if(Debug) { dbgprintf("--------imUDP calling select, active file descriptors (max %d): ", maxfds); for (nfds = 0; nfds <= maxfds; ++nfds) - if ( FD_ISSET(nfds, &readfds) ) + if ( FD_ISSET(nfds, pReadfds) ) dbgprintf("%d ", nfds); dbgprintf("\n"); } @@ -443,10 +442,6 @@ rsRetVal rcvMainLoop(thrdInfo_t *pThrd) */ BEGINrunInput CODESTARTrunInput - /* this is an endless loop - it is terminated when the thread is - * signalled to do so. This, however, is handled by the framework, - * right into the sleep below. - */ iRet = rcvMainLoop(pThrd); ENDrunInput @@ -460,6 +455,7 @@ CODESTARTwillRun CHKiRet(prop.ConstructFinalize(pInputName)); net.PrintAllowedSenders(1); /* UDP */ + net.HasRestrictions(UCHAR_CONSTANT("UDP"), &bDoACLCheck); /* UDP */ /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 17c85afb..8ad2b9e9 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -46,6 +46,7 @@ #include "msg.h" #include "prop.h" #include "debug.h" +#include "unlimited_select.h" MODULE_TYPE_INPUT @@ -294,7 +295,13 @@ BEGINrunInput int nfds; int i; int fd; - fd_set readfds; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = malloc(glbl.GetFdSetSize()); +#else + fd_set readfds; + fd_set *pReadfds = &readfds; +#endif + CODESTARTrunInput /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, @@ -308,11 +315,11 @@ CODESTARTrunInput * is given without -a, we do not need to listen at all.. */ maxfds = 0; - FD_ZERO (&readfds); + FD_ZERO (pReadfds); /* Copy master connections */ for (i = startIndexUxLocalSockets; i < nfunix; i++) { if (funix[i] != -1) { - FD_SET(funix[i], &readfds); + FD_SET(funix[i], pReadfds); if (funix[i]>maxfds) maxfds=funix[i]; } } @@ -320,20 +327,20 @@ CODESTARTrunInput if(Debug) { dbgprintf("--------imuxsock calling select, active file descriptors (max %d): ", maxfds); for (nfds= 0; nfds <= maxfds; ++nfds) - if ( FD_ISSET(nfds, &readfds) ) + if ( FD_ISSET(nfds, pReadfds) ) dbgprintf("%d ", nfds); dbgprintf("\n"); } /* wait for io to become ready */ - nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); + nfds = select(maxfds+1, (fd_set *) pReadfds, NULL, NULL, NULL); if(glbl.GetGlobalInputTermState() == 1) break; /* terminate input! */ for (i = 0; i < nfunix && nfds > 0; i++) { if(glbl.GetGlobalInputTermState() == 1) ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ - if ((fd = funix[i]) != -1 && FD_ISSET(fd, &readfds)) { + if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) { readSocket(fd, i); --nfds; /* indicate we have processed one */ } @@ -341,6 +348,7 @@ CODESTARTrunInput } finalize_it: + freeFdSet(pReadfds); RETiRet; ENDrunInput diff --git a/plugins/omdbalerting/Makefile.am b/plugins/omdbalerting/Makefile.am new file mode 100644 index 00000000..becf29b0 --- /dev/null +++ b/plugins/omdbalerting/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omdbalerting.la + +omdbalerting_la_SOURCES = omdbalerting.c +omdbalerting_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omdbalerting_la_LDFLAGS = -module -avoid-version +omdbalerting_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omdbalerting/omdbalerting.c b/plugins/omdbalerting/omdbalerting.c new file mode 100644 index 00000000..2e04391c --- /dev/null +++ b/plugins/omdbalerting/omdbalerting.c @@ -0,0 +1,144 @@ +/* omdbalerting.c + * generate alerts based on database contents - so far a skeleton + * left for implementation by somebody else (skeleton created on request). + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-11-17 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* config variables */ + + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omdbalerting:", sizeof(":dbalerting:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omdbalerting:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + /* we request the standard interface via template, others may be more useful + * here. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + // SAMPLE! CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomdbalertingensurelfending", 0, eCmdHdlrBinary, NULL, + // &bEnsureLFEnding, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index caf7c5ca..9047c83d 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -16,6 +16,7 @@ librsyslog_la_SOURCES = \ nsd.h \ glbl.h \ glbl.c \ + unlimited_select.h \ conf.c \ conf.h \ parser.h \ @@ -136,7 +137,10 @@ lmnet_la_LDFLAGS = -module -avoid-version lmnet_la_LIBADD = # network stream master class and stream factory -lmnetstrms_la_SOURCES = netstrms.c netstrms.h netstrm.c netstrm.h nssel.c nssel.h +lmnetstrms_la_SOURCES = netstrms.c netstrms.h \ + netstrm.c netstrm.h \ + nssel.c nssel.h \ + nspoll.c nspoll.h lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version lmnetstrms_la_LIBADD = @@ -152,7 +156,9 @@ lmstrmsrv_la_LIBADD = # plain tcp driver - main driver pkglib_LTLIBRARIES += lmnsd_ptcp.la -lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h nsdsel_ptcp.c nsdsel_ptcp.h +lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h \ + nsdsel_ptcp.c nsdsel_ptcp.h \ + nsdpoll_ptcp.c nsdpoll_ptcp.h lmnsd_ptcp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnsd_ptcp_la_LDFLAGS = -module -avoid-version lmnsd_ptcp_la_LIBADD = diff --git a/runtime/debug.c b/runtime/debug.c index 545ac876..6d82397f 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -47,6 +47,9 @@ #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif #include "rsyslog.h" #include "debug.h" @@ -844,6 +847,9 @@ do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg) char pszWriteBuf[32*1024]; size_t lenWriteBuf; struct timespec t; +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif /* The bWasNL handler does not really work. It works if no thread * switching occurs during non-NL messages. Else, things are messed @@ -869,7 +875,14 @@ do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg) if(bWasNL) { if(bPrintTime) { +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%4.4ld.%9.9ld:", (long) (t.tv_sec % 10000), t.tv_nsec); if(stddbg != -1) write(stddbg, pszWriteBuf, lenWriteBuf); @@ -1308,11 +1321,11 @@ dbgGetRuntimeOptions(void) /* this is earlier in the process than the -d option, as such it * allows us to spit out debug messages from the very beginning. */ - Debug = 1; + Debug = DEBUG_FULL; debugging_on = 1; } else if(!strcasecmp((char*)optname, "debugondemand")) { /* Enables debugging, but turns off debug output */ - Debug = 1; + Debug = DEBUG_ONDEMAND; debugging_on = 1; dbgprintf("Note: debug on demand turned on via configuraton file, " "use USR1 signal to activate.\n"); diff --git a/runtime/debug.h b/runtime/debug.h index 8d9c1ceb..c011dd2d 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -29,6 +29,11 @@ #include <pthread.h> #include "obj-types.h" +/* some settings for various debug modes */ +#define DEBUG_OFF 0 +#define DEBUG_ONDEMAND 1 +#define DEBUG_FULL 2 + /* external static data elements (some time to be replaced) */ extern int Debug; /* debug flag - read-only after startup */ extern int debugging_on; /* read-only, except on sig USR1 */ diff --git a/runtime/glbl.c b/runtime/glbl.c index 71c2ed0d..ac08791f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -74,6 +74,9 @@ static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm d static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */ static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */ static int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ +#ifdef USE_UNLIMITED_SELECT +static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */ +#endif /* define a macro for the simple properties' set and get functions @@ -106,6 +109,9 @@ SIMP_PROP(DisableDNS, bDisableDNS, int) SIMP_PROP(LocalDomain, LocalDomain, uchar*) SIMP_PROP(StripDomains, StripDomains, char**) SIMP_PROP(LocalHosts, LocalHosts, char**) +#ifdef USE_UNLIMITED_SELECT +SIMP_PROP(FdSetSize, iFdSetSize, int) +#endif SIMP_PROP_SET(LocalFQDNName, LocalFQDNName, uchar*) SIMP_PROP_SET(LocalHostName, LocalHostName, uchar*) @@ -284,6 +290,9 @@ CODESTARTobjQueryInterface(glbl) SIMP_PROP(DfltNetstrmDrvrCAF) SIMP_PROP(DfltNetstrmDrvrKeyFile) SIMP_PROP(DfltNetstrmDrvrCertFile) +#ifdef USE_UNLIMITED_SELECT + SIMP_PROP(FdSetSize) +#endif #undef SIMP_PROP finalize_it: ENDobjQueryInterface(glbl) @@ -317,6 +326,9 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; bPreserveFQDN = 0; +#ifdef USE_UNLIMITED_SELECT + iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); +#endif return RS_RET_OK; } diff --git a/runtime/glbl.h b/runtime/glbl.h index 7506f16b..4b4bdf83 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -66,9 +66,20 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ void (*SetGlobalInputTermination)(void); /* added v5, 2009-11-03 */ SIMP_PROP(ParseHOSTNAMEandTAG, int) + /* note: v4, v5 are already used by more recent versions, so we need to skip them! */ + /* added v6, 2009-11-16 as part of varmojfekoj's "unlimited select()" patch + * Note that it must be always present, otherwise the interface would have different + * versions depending on compile settings, what is not acceptable. + * Use this property with care, it is only truly available if UNLIMITED_SELECT is enabled + * (I did not yet further investigate the details, because that code hopefully can be removed + * at some later stage). + */ + SIMP_PROP(FdSetSize, int) + /* v7: was neeeded to mean v5+v6 - do NOT add anything else for that version! */ + /* next change is v8! */ #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ diff --git a/runtime/modules.c b/runtime/modules.c index fd3468d8..1af94abc 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -472,7 +472,6 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction); if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) { pNew->mod.om.endTransaction = dummyEndTransaction; - //pNew->mod.om.beginTransaction = dummyEndTransaction; } else if(localRet != RS_RET_OK) { ABORT_FINALIZE(localRet); } @@ -559,10 +558,35 @@ static void modPrintList(void) dbgprintf(" module.\n"); dbgprintf("Entry points:\n"); dbgprintf("\tqueryEtryPt: 0x%lx\n", (unsigned long) pMod->modQueryEtryPt); - dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); - dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); dbgprintf("\tdbgPrintInstInfo: 0x%lx\n", (unsigned long) pMod->dbgPrintInstInfo); dbgprintf("\tfreeInstance: 0x%lx\n", (unsigned long) pMod->freeInstance); + switch(pMod->eType) { + case eMOD_OUT: + dbgprintf("Output Module Entry Points:\n"); + dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); + dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); + dbgprintf("\ttryResume: 0x%lx\n", (unsigned long) pMod->tryResume); + dbgprintf("\tdoHUP: 0x%lx\n", (unsigned long) pMod->doHUP); + dbgprintf("\tBeginTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.beginTransaction == dummyBeginTransaction) ? + 0 : pMod->mod.om.beginTransaction)); + dbgprintf("\tEndTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.endTransaction == dummyEndTransaction) ? + 0 : pMod->mod.om.endTransaction)); + break; + case eMOD_IN: + dbgprintf("Input Module Entry Points\n"); + dbgprintf("\trunInput: 0x%lx\n", (unsigned long) pMod->mod.im.runInput); + dbgprintf("\twillRun: 0x%lx\n", (unsigned long) pMod->mod.im.willRun); + dbgprintf("\tafterRun: 0x%lx\n", (unsigned long) pMod->mod.im.afterRun); + break; + case eMOD_LIB: + break; + case eMOD_PARSER: + dbgprintf("Parser Module Entry Points\n"); + dbgprintf("\tparse: 0x%lx\n", (unsigned long) pMod->mod.pm.parse); + break; + } dbgprintf("\n"); pMod = GetNxt(pMod); /* done, go next */ } diff --git a/runtime/msg.c b/runtime/msg.c index 2fd4cff7..629c6f24 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -35,6 +35,8 @@ #include <string.h> #include <assert.h> #include <ctype.h> +#include <sys/socket.h> +#include <netdb.h> #if HAVE_MALLOC_H # include <malloc.h> #endif @@ -51,6 +53,7 @@ #include "unicode-helper.h" #include "ruleset.h" #include "prop.h" +#include "net.h" /* static data */ DEFobjStaticHelpers @@ -59,6 +62,7 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) DEFobjCurrIf(prop) +DEFobjCurrIf(net) static struct { uchar *pszName; @@ -275,7 +279,7 @@ static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "15", "16", "17", "18", "19", "20", "21", "22", "23" }; /* some forward declarations */ -static int getAPPNAMELen(msg_t *pM, bool bLockMutex); +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); static inline int getProtocolVersion(msg_t *pM) @@ -284,6 +288,41 @@ static inline int getProtocolVersion(msg_t *pM) } +/* do a DNS reverse resolution, if not already done, reflect status + * rgerhards, 2009-11-16 + */ +static inline rsRetVal +resolveDNS(msg_t *pMsg) { + rsRetVal localRet; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + DEFiRet; + + CHKiRet(objUse(net, CORE_COMPONENT)); + if(pMsg->msgFlags & NEEDS_DNSRESOL) { + localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); + if(localRet == RS_RET_OK) { + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + } + } +finalize_it: + if(iRet != RS_RET_OK) { + /* best we can do: remove property */ + MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); + prop.Destruct(&propFromHost); + } + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; +} + + static inline void getInputName(msg_t *pM, uchar **ppsz, int *plen) { @@ -307,6 +346,7 @@ getRcvFromIP(msg_t *pM) if(pM == NULL) { psz = UCHAR_CONSTANT(""); } else { + resolveDNS(pM); /* make sure we have a resolved entry */ if(pM->pRcvFromIP == NULL) psz = UCHAR_CONSTANT(""); else @@ -660,7 +700,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pCSMSGID = NULL; pM->pInputName = NULL; pM->pRcvFromIP = NULL; - pM->pRcvFrom = NULL; + pM->rcvFrom.pRcvFrom = NULL; pM->pRuleset = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); @@ -761,8 +801,12 @@ CODESTARTobjDestruct(msg) freeHOSTNAME(pThis); if(pThis->pInputName != NULL) prop.Destruct(&pThis->pInputName); - if(pThis->pRcvFrom != NULL) - prop.Destruct(&pThis->pRcvFrom); + if((pThis->msgFlags & NEEDS_DNSRESOL) == 0) { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } else { + free(pThis->rcvFrom.pfrominet); + } if(pThis->pRcvFromIP != NULL) prop.Destruct(&pThis->pRcvFromIP); free(pThis->pszRcvdAt3164); @@ -848,6 +892,7 @@ ENDobjDestruct(msg) msg_t* MsgDup(msg_t* pOld) { msg_t* pNew; + rsRetVal localRet; assert(pOld != NULL); @@ -868,9 +913,19 @@ msg_t* MsgDup(msg_t* pOld) pNew->iLenMSG = pOld->iLenMSG; pNew->iLenTAG = pOld->iLenTAG; pNew->iLenHOSTNAME = pOld->iLenHOSTNAME; - if(pOld->pRcvFrom != NULL) { - pNew->pRcvFrom = pOld->pRcvFrom; - prop.AddRef(pNew->pRcvFrom); + if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) { + localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet); + if(localRet != RS_RET_OK) { + /* if something fails, we accept loss of this property, it is + * better than losing the whole message. + */ + pNew->msgFlags &= ~NEEDS_DNSRESOL; + } + } else { + if(pOld->rcvFrom.pRcvFrom != NULL) { + pNew->rcvFrom.pRcvFrom = pOld->rcvFrom.pRcvFrom; + prop.AddRef(pNew->rcvFrom.pRcvFrom); + } } if(pOld->pRcvFromIP != NULL) { pNew->pRcvFromIP = pOld->pRcvFromIP; @@ -1488,7 +1543,7 @@ finalize_it: * This must be called WITHOUT the message lock being held. * rgerhards, 2009-06-26 */ -static inline void preparePROCID(msg_t *pM, bool bLockMutex) +static inline void preparePROCID(msg_t *pM, sbool bLockMutex) { if(pM->pCSPROCID == NULL) { if(bLockMutex == LOCK_MUTEX) @@ -1505,7 +1560,7 @@ static inline void preparePROCID(msg_t *pM, bool bLockMutex) #if 0 /* rgerhards, 2005-11-24 */ -static inline int getPROCIDLen(msg_t *pM, bool bLockMutex) +static inline int getPROCIDLen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); preparePROCID(pM, bLockMutex); @@ -1516,7 +1571,7 @@ static inline int getPROCIDLen(msg_t *pM, bool bLockMutex) /* rgerhards, 2005-11-24 */ -char *getPROCID(msg_t *pM, bool bLockMutex) +char *getPROCID(msg_t *pM, sbool bLockMutex) { ISOBJ_TYPE_assert(pM, msg); preparePROCID(pM, bLockMutex); @@ -1595,7 +1650,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) * if there is a TAG and, if not, if it can emulate it. * rgerhards, 2005-11-24 */ -static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex) +static inline void tryEmulateTAG(msg_t *pM, sbool bLockMutex) { size_t lenTAG; uchar bufTAG[CONF_TAG_MAXSIZE]; @@ -1648,12 +1703,13 @@ int getHOSTNAMELen(msg_t *pM) if(pM == NULL) return 0; else - if(pM->pszHOSTNAME == NULL) - if(pM->pRcvFrom == NULL) + if(pM->pszHOSTNAME == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) return 0; else - return prop.GetStringLen(pM->pRcvFrom); - else + return prop.GetStringLen(pM->rcvFrom.pRcvFrom); + } else return pM->iLenHOSTNAME; } @@ -1664,12 +1720,13 @@ char *getHOSTNAME(msg_t *pM) return ""; else if(pM->pszHOSTNAME == NULL) { - if(pM->pRcvFrom == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) { return ""; } else { uchar *psz; int len; - prop.GetString(pM->pRcvFrom, &psz, &len); + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); return (char*) psz; } } else { @@ -1683,13 +1740,15 @@ uchar *getRcvFrom(msg_t *pM) uchar *psz; int len; BEGINfunc + if(pM == NULL) { psz = UCHAR_CONSTANT(""); } else { - if(pM->pRcvFrom == NULL) + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) psz = UCHAR_CONSTANT(""); else - prop.GetString(pM->pRcvFrom, &psz, &len); + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); } ENDfunc return psz; @@ -1736,7 +1795,7 @@ static inline char *getStructuredData(msg_t *pM) /* check if we have a ProgramName, and, if not, try to aquire/emulate it. * rgerhards, 2009-06-26 */ -static inline void prepareProgramName(msg_t *pM, bool bLockMutex) +static inline void prepareProgramName(msg_t *pM, sbool bLockMutex) { if(pM->pCSProgName == NULL) { if(bLockMutex == LOCK_MUTEX) @@ -1755,7 +1814,7 @@ static inline void prepareProgramName(msg_t *pM, bool bLockMutex) /* get the length of the "programname" sz string * rgerhards, 2005-10-19 */ -int getProgramNameLen(msg_t *pM, bool bLockMutex) +int getProgramNameLen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); prepareProgramName(pM, bLockMutex); @@ -1766,7 +1825,7 @@ int getProgramNameLen(msg_t *pM, bool bLockMutex) /* get the "programname" as sz string * rgerhards, 2005-10-19 */ -uchar *getProgramName(msg_t *pM, bool bLockMutex) +uchar *getProgramName(msg_t *pM, sbool bLockMutex) { prepareProgramName(pM, bLockMutex); return (pM->pCSProgName == NULL) ? UCHAR_CONSTANT("") : rsCStrGetSzStrNoNULL(pM->pCSProgName); @@ -1796,7 +1855,7 @@ static void tryEmulateAPPNAME(msg_t *pM) * This must be called WITHOUT the message lock being held. * rgerhards, 2009-06-26 */ -static inline void prepareAPPNAME(msg_t *pM, bool bLockMutex) +static inline void prepareAPPNAME(msg_t *pM, sbool bLockMutex) { if(pM->pCSAPPNAME == NULL) { if(bLockMutex == LOCK_MUTEX) @@ -1813,7 +1872,7 @@ static inline void prepareAPPNAME(msg_t *pM, bool bLockMutex) /* rgerhards, 2005-11-24 */ -char *getAPPNAME(msg_t *pM, bool bLockMutex) +char *getAPPNAME(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); prepareAPPNAME(pM, bLockMutex); @@ -1822,7 +1881,7 @@ char *getAPPNAME(msg_t *pM, bool bLockMutex) /* rgerhards, 2005-11-24 */ -static int getAPPNAMELen(msg_t *pM, bool bLockMutex) +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); prepareAPPNAME(pM, bLockMutex); @@ -1845,6 +1904,28 @@ void MsgSetInputName(msg_t *pThis, prop_t *inputName) } +/* Set the pfrominet socket store, so that we can obtain the peer at some + * later time. Note that we do not check if pRcvFrom is already set, so this + * function must only be called during message creation. + * NOTE: msgFlags is NOT set. While this is somewhat a violation of layers, + * it is done because it gains us some performance. So the caller must make + * sure the message flags are properly maintained. For all current callers, + * this is always the case and without extra effort required. + * rgerhards, 2009-11-17 + */ +rsRetVal +msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa){ + DEFiRet; + assert(pThis->rcvFrom.pRcvFrom == NULL); + + CHKmalloc(pThis->rcvFrom.pfrominet = malloc(sizeof(struct sockaddr_storage))); + memcpy(pThis->rcvFrom.pfrominet, sa, sizeof(struct sockaddr_storage)); + +finalize_it: + RETiRet; +} + + /* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef() * on the property, because this must be done in all current cases and there * is no case expected where this may not be necessary. @@ -1855,9 +1936,15 @@ void MsgSetRcvFrom(msg_t *pThis, prop_t *new) assert(pThis != NULL); prop.AddRef(new); - if(pThis->pRcvFrom != NULL) - prop.Destruct(&pThis->pRcvFrom); - pThis->pRcvFrom = new; + if(pThis->msgFlags & NEEDS_DNSRESOL) { + if(pThis->rcvFrom.pfrominet != NULL) + free(pThis->rcvFrom.pfrominet); + pThis->msgFlags &= ~NEEDS_DNSRESOL; + } else { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } + pThis->rcvFrom.pRcvFrom = new; } diff --git a/runtime/msg.h b/runtime/msg.h index 9101cef7..b4b6d9f8 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -59,14 +59,8 @@ struct msg { flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ pthread_mutex_t mut; - bool bDoLock; /* use the mutex? */ + sbool bDoLock; /* use the mutex? */ short iRefCount; /* reference counter (0 = unused) */ - /* background: the hostname is not present on "regular" messages - * received via UNIX domain sockets from the same machine. However, - * it is available when we have a forwarder (e.g. rfc3195d) using local - * sockets. All in all, the parser would need parse templates, that would - * resolve all these issues... rgerhards, 2005-10-06 - */ short iSeverity; /* the severity 0..7 */ short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ @@ -94,8 +88,12 @@ struct msg { cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ prop_t *pInputName; /* input name property */ - prop_t *pRcvFrom; /* name of system message was received from */ prop_t *pRcvFromIP; /* IP of system message was received from */ + union { + prop_t *pRcvFrom;/* name of system message was received from */ + struct sockaddr_storage *pfrominet; /* unresolved name */ + } rcvFrom; + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -129,6 +127,8 @@ struct msg { #define MARK 0x008 /* this message is a mark */ #define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ #define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ +#define NEEDS_DNSRESOL 0x040 /* fromhost address is unresolved and must be locked up via DNS reverse lookup first */ +#define NEEDS_ACLCHK_U 0x080 /* check UDP ACLs after DNS resolution has been done in main queue consumer */ /* function prototypes @@ -148,6 +148,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); +rsRetVal msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa); void MsgSetRcvFrom(msg_t *pMsg, prop_t*); void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); @@ -169,14 +170,14 @@ uchar *getRcvFrom(msg_t *pM); /* TODO: remove these five (so far used in action.c) */ uchar *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); -char *getPROCID(msg_t *pM, bool bLockMutex); -char *getAPPNAME(msg_t *pM, bool bLockMutex); +char *getPROCID(msg_t *pM, sbool bLockMutex); +char *getAPPNAME(msg_t *pM, sbool bLockMutex); int getMSGLen(msg_t *pM); char *getHOSTNAME(msg_t *pM); int getHOSTNAMELen(msg_t *pM); -uchar *getProgramName(msg_t *pM, bool bLockMutex); -int getProgramNameLen(msg_t *pM, bool bLockMutex); +uchar *getProgramName(msg_t *pM, sbool bLockMutex); +int getProgramNameLen(msg_t *pM, sbool bLockMutex); uchar *getRcvFrom(msg_t *pM); rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); uchar *propIDToName(propid_t propID); diff --git a/runtime/net.c b/runtime/net.c index f52d408c..ab431f7c 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -892,15 +892,18 @@ rsRetVal addAllowedSenderLine(char* pName, uchar** ppRestOfConfLine) * including IPv4/v6 as well as domain name wildcards. * This is a helper to isAllowedSender. As it is only called once, it is * declared inline. - * Returns 0 if they do not match, something else otherwise. - * contributed 1007-07-16 by mildew@gmail.com + * Returns 0 if they do not match, 1 if they match and 2 if a DNS name would have been required. + * contributed 2007-07-16 by mildew@gmail.com */ -static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost) +static inline int +MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { assert(pAllow != NULL); assert(pFrom != NULL); if(F_ISSET(pAllow->flags, ADDR_NAME)) { + if(bChkDNS == 0) + return 2; dbgprintf("MaskCmp: host=\"%s\"; pattern=\"%s\"\n", pszFromHost, pAllow->addr.HostWildcard); # if !defined(FNM_CASEFOLD) @@ -967,18 +970,22 @@ static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr /* check if a sender is allowed. The root of the the allowed sender. * list must be proveded by the caller. As such, this function can be * used to check both UDP and TCP allowed sender lists. - * returns 1, if the sender is allowed, 0 otherwise. + * returns 1, if the sender is allowed, 0 if not and 2 if we could not + * obtain a result because we would need a dns name, which we don't have + * (2 was added rgerhards, 2009-11-16). * rgerhards, 2005-09-26 */ -static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) +static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { struct AllowedSenders *pAllow; struct AllowedSenders *pAllowRoot; + int bNeededDNS = 0; /* partial check because we could not resolve DNS? */ + int ret; assert(pFrom != NULL); if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK) - return 0; /* if something went wrong, we denie access - that's the better choice... */ + return 0; /* if something went wrong, we deny access - that's the better choice... */ if(pAllowRoot == NULL) return 1; /* checking disabled, everything is valid! */ @@ -990,10 +997,20 @@ static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *p * that the sender is disallowed. */ for(pAllow = pAllowRoot ; pAllow != NULL ; pAllow = pAllow->pNext) { - if (MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost)) + ret = MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost, bChkDNS); + if(ret == 1) return 1; + else if(ret == 2) + bNeededDNS = 2; } - return 0; + return bNeededDNS; +} + + +/* legacy API, not to be used any longer */ +static int +isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) { + return isAllowedSender2(pszType, pFrom, pszFromHost, 1); } @@ -1533,12 +1550,36 @@ static int CmpHost(struct sockaddr_storage *s1, struct sockaddr_storage* s2, siz ret = memcmp(s1, s2, socklen); } -dbgprintf("CmpHost returns %d\n", ret); finalize_it: return ret; } + +/* check if restrictions (ALCs) exists. The goal of this function is to disable the + * somewhat time-consuming ACL checks if no restrictions are defined (the usual case). + * This also permits to gain some speedup by using firewall-based ACLs instead of + * rsyslog ACLs (the recommended method. + * rgerhards, 2009-11-16 + */ +static rsRetVal +HasRestrictions(uchar *pszType, int *bHasRestrictions) { + struct AllowedSenders *pAllowRoot; + DEFiRet; + + CHKiRet(setAllowRoot(&pAllowRoot, pszType)); + + *bHasRestrictions = (pAllowRoot == NULL) ? 0 : 1; + +finalize_it: + if(iRet != RS_RET_OK) { + *bHasRestrictions = 1; /* in this case it is better to check individually */ + DBGPRINTF("Error %d trying to obtain ACL restriction state of '%s'\n", iRet, pszType); + } + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-03-05 */ @@ -1562,12 +1603,14 @@ CODESTARTobjQueryInterface(net) pIf->create_udp_socket = create_udp_socket; pIf->closeUDPListenSockets = closeUDPListenSockets; pIf->isAllowedSender = isAllowedSender; + pIf->isAllowedSender2 = isAllowedSender2; pIf->should_use_so_bsdcompat = should_use_so_bsdcompat; pIf->getLocalHostname = getLocalHostname; pIf->AddPermittedPeer = AddPermittedPeer; pIf->DestructPermittedPeers = DestructPermittedPeers; pIf->PermittedPeerWildcardMatch = PermittedPeerWildcardMatch; pIf->CmpHost = CmpHost; + pIf->HasRestrictions = HasRestrictions; /* data members */ pIf->pACLAddHostnameOnFail = &ACLAddHostnameOnFail; pIf->pACLDontResolve = &ACLDontResolve; diff --git a/runtime/net.h b/runtime/net.h index ec364b1c..101ce79d 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -139,7 +139,7 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ void (*debugListenInfo)(int fd, char *type); int *(*create_udp_socket)(uchar *hostname, uchar *LogPort, int bIsServer); void (*closeUDPListenSockets)(int *finet); - int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); + int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); /* deprecated! */ rsRetVal (*getLocalHostname)(uchar**); int (*should_use_so_bsdcompat)(void); /* permitted peer handling should be replaced by something better (see comments above) */ @@ -148,11 +148,14 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ rsRetVal (*PermittedPeerWildcardMatch)(permittedPeers_t *pPeer, uchar *pszNameToMatch, int *pbIsMatching); /* v5 interface additions */ int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t); + /* v6 interface additions - 2009-11-16 */ + rsRetVal (*HasRestrictions)(uchar *, int *bHasRestrictions); + int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS); /* data members - these should go away over time... TODO */ int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ ENDinterface(net) -#define netCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define netCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* prototypes */ PROTOTYPEObj(net); diff --git a/runtime/netstrms.c b/runtime/netstrms.c index 6b28e7ea..e9ff2568 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -36,6 +36,7 @@ #include "nsd.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "netstrms.h" MODULE_TYPE_LIB @@ -304,6 +305,7 @@ ENDObjClassInit(netstrms) BEGINmodExit CODESTARTmodExit nsselClassExit(); + nspollClassExit(); netstrmsClassExit(); netstrmClassExit(); /* we use this object, so we must exit it after we are finished */ ENDmodExit @@ -322,6 +324,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(netstrmClassInit(pModInfo)); CHKiRet(nsselClassInit(pModInfo)); + CHKiRet(nspollClassInit(pModInfo)); CHKiRet(netstrmsClassInit(pModInfo)); ENDmodInit /* vi:set ai: diff --git a/runtime/nsd.h b/runtime/nsd.h index 8668c934..e5b9320b 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -87,4 +87,13 @@ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ ENDinterface(nsdsel) #define nsdselCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface for the epoll call */ +BEGINinterface(nsdpoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nsdpoll_t **ppThis); + rsRetVal (*Destruct)(nsdpoll_t **ppThis); + rsRetVal (*Ctl)(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op); + rsRetVal (*Wait)(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); +ENDinterface(nsdpoll) +#define nsdpollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + #endif /* #ifndef INCLUDED_NSD_H */ diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index fe31ab40..744955c7 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -48,6 +48,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nsdsel_ptcp.h" +#include "nsdpoll_ptcp.h" #include "nsd_ptcp.h" MODULE_TYPE_LIB @@ -562,6 +563,7 @@ finalize_it: static rsRetVal Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) { + char errStr[1024]; DEFiRet; nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; ISOBJ_TYPE_assert(pThis, nsd_ptcp); @@ -571,7 +573,9 @@ Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) if(*pLenBuf == 0) { ABORT_FINALIZE(RS_RET_CLOSED); } else if (*pLenBuf < 0) { - ABORT_FINALIZE(RS_RET_ERR); + rs_strerror_r(errno, errStr, sizeof(errStr)); + dbgprintf("error during recv on NSD %p: %s\n", pNsd, errStr); + ABORT_FINALIZE(RS_RET_RCV_ERR); } finalize_it: @@ -821,6 +825,7 @@ ENDObjClassInit(nsd_ptcp) BEGINmodExit CODESTARTmodExit + nsdpoll_ptcpClassExit(); nsdsel_ptcpClassExit(); nsd_ptcpClassExit(); ENDmodExit @@ -839,6 +844,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(nsd_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ CHKiRet(nsdsel_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + CHKiRet(nsdpoll_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ ENDmodInit /* vi:set ai: */ diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c new file mode 100644 index 00000000..85aac04c --- /dev/null +++ b/runtime/nsdpoll_ptcp.c @@ -0,0 +1,284 @@ +/* nsdpoll_ptcp.c + * + * An implementation of the nsd epoll() interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "errmsg.h" +#include "srUtils.h" +#include "nspoll.h" +#include "nsd_ptcp.h" +#include "nsdpoll_ptcp.h" +#include "unlimited_select.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + + +/* -START------------------------- helpers for event list ------------------------------------ */ + +/* add new entry to list. We assume that the fd is not already present and DO NOT check this! + * Returns newly created entry in pEvtLst. + * Note that we currently need to use level-triggered mode, because the upper layers do not work + * in parallel. As such, in edge-triggered mode we may not get notified, because new data comes + * in after we have read everything that was present. To use ET mode, we need to change the upper + * peers so that they immediately start a new wait before processing the data read. That obviously + * requires more elaborate redesign and we postpone this until the current more simplictic mode has + * been proven OK in practice. + * rgerhards, 2009-11-18 + */ +static inline rsRetVal +addEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, int mode, nsd_ptcp_t *pSock, nsdpoll_epollevt_lst_t **pEvtLst) { + nsdpoll_epollevt_lst_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (nsdpoll_epollevt_lst_t*) malloc(sizeof(nsdpoll_epollevt_lst_t))); + pNew->id = id; + pNew->pUsr = pUsr; + pNew->pSock = pSock; + pNew->event.events = 0; /* TODO: at some time we should be able to use EPOLLET */ + if(mode & NSDPOLL_IN) + pNew->event.events |= EPOLLIN; + if(mode & NSDPOLL_OUT) + pNew->event.events |= EPOLLOUT; + pNew->event.data.u64 = (uint64) pNew; + pNew->pNext = pThis->pRoot; + pThis->pRoot = pNew; + *pEvtLst = pNew; + +finalize_it: + RETiRet; +} + + +/* find and unlink the entry identified by id/pUsr from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +unlinkEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, nsdpoll_epollevt_lst_t **ppEvtLst) { + nsdpoll_epollevt_lst_t *pEvtLst; + nsdpoll_epollevt_lst_t *pPrev = NULL; + DEFiRet; + + pEvtLst = pThis->pRoot; + while(pEvtLst != NULL && !(pEvtLst->id == id && pEvtLst->pUsr == pUsr)) { + pPrev = pEvtLst; + pEvtLst = pEvtLst->pNext; + } + if(pEvtLst == NULL) + ABORT_FINALIZE(RS_RET_NOT_FOUND); + + *ppEvtLst = pEvtLst; + + /* unlink */ + if(pPrev == NULL) + pThis->pRoot = pEvtLst->pNext; + else + pPrev->pNext = pEvtLst->pNext; + +finalize_it: + RETiRet; +} + + +/* destruct the provided element. It must already be unlinked from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) { + DEFiRet; + free(*ppEvtLst); + *ppEvtLst = NULL; + RETiRet; +} + + +/* -END--------------------------- helpers for event list ------------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */ +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); + pThis->efd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n"); + pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */ +# endif + if(pThis->efd < 0) { + DBGPRINTF("epoll_create1() could not create fd\n"); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } +finalize_it: +ENDobjConstruct(nsdpoll_ptcp) + + +/* destructor for the nsdpoll_ptcp object */ +BEGINobjDestruct(nsdpoll_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nsdpoll_ptcp) +ENDobjDestruct(nsdpoll_ptcp) + + +/* Modify socket set */ +static rsRetVal +Ctl(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; + nsdpoll_epollevt_lst_t *pEventLst; + int errSave; + char errStr[512]; + DEFiRet; + + if(op == NSDPOLL_ADD) { + dbgprintf("adding nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(addEvent(pThis, id, pUsr, mode, pSock, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_ADD, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + } + } else if(op == NSDPOLL_DEL) { + dbgprintf("removing nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(unlinkEvent(pThis, id, pUsr, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_DEL, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + ABORT_FINALIZE(RS_RET_ERR_EPOLL_CTL); + } + CHKiRet(delEvent(&pEventLst)); + } else { + dbgprintf("program error: invalid NSDPOLL_mode %d - ignoring request\n", op); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Wait for io to become ready. After the successful call, idRdy contains the + * id set by the caller for that i/o event, ppUsr is a pointer to a location + * where the user pointer shall be stored. + * TODO: this is a trivial implementation that only polls one event at a time. We + * may later extend it to poll for multiple events, what would cause less + * overhead. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Wait(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsdpoll_epollevt_lst_t *pOurEvt; + struct epoll_event event; + int nfds; + DEFiRet; + + assert(idRdy != NULL); + assert(ppUsr != NULL); + + nfds = epoll_wait(pThis->efd, &event, 1, timeout); + if(nfds == -1) { + if(errno == EINTR) { + ABORT_FINALIZE(RS_RET_EINTR); + } else { + DBGPRINTF("epoll() returned with error code %d\n", errno); + ABORT_FINALIZE(RS_RET_ERR_EPOLL); + } + } else if(nfds == 0) { + ABORT_FINALIZE(RS_RET_TIMEOUT); + } + + /* we got a valid event, so tell the caller... */ + pOurEvt = (nsdpoll_epollevt_lst_t*) event.data.u64; + *idRdy = pOurEvt->id; + *ppUsr = pOurEvt->pUsr; + +finalize_it: + RETiRet; +} + + +/* ------------------------------ end support for the epoll() interface ------------------------------ */ + + +/* queryInterface function */ +BEGINobjQueryInterface(nsdpoll_ptcp) +CODESTARTobjQueryInterface(nsdpoll_ptcp) + if(pIf->ifVersion != nsdCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpConstruct; + pIf->Destruct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpDestruct; + pIf->Ctl = Ctl; + pIf->Wait = Wait; +finalize_it: +ENDobjQueryInterface(nsdpoll_ptcp) + + +/* exit our class + */ +BEGINObjClassExit(nsdpoll_ptcp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdpoll_ptcp) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(nsdpoll_ptcp) + + +/* Initialize the nsdpoll_ptcp class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nsdpoll_ptcp, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nsdpoll_ptcp) +/* vi:set ai: + */ diff --git a/runtime/nsdpoll_ptcp.h b/runtime/nsdpoll_ptcp.h new file mode 100644 index 00000000..0708e489 --- /dev/null +++ b/runtime/nsdpoll_ptcp.h @@ -0,0 +1,60 @@ +/* An implementation of the nsd poll interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSDPOLL_PTCP_H +#define INCLUDED_NSDPOLL_PTCP_H + +#include "nsd.h" +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#else + typedef void epoll_event_t; +#endif +typedef nsdpoll_if_t nsdpoll_ptcp_if_t; /* we just *implement* this interface */ +/* a helper object to keep track of the epoll event records + * Note that we need to keep track of that list because we need to + * free the events when they are no longer needed. + */ +typedef struct nsdpoll_epollevt_lst_s nsdpoll_epollevt_lst_t; +struct nsdpoll_epollevt_lst_s { + epoll_event_t event; + int id; + void *pUsr; + nsd_ptcp_t *pSock; /* our associated netstream driver data */ + nsdpoll_epollevt_lst_t *pNext; +}; + +/* the nsdpoll_ptcp object */ +struct nsdpoll_ptcp_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int efd; /* file descriptor used by epoll */ + nsdpoll_epollevt_lst_t *pRoot; /* Root of the epoll event list */ +}; + +/* interface is defined in nsd.h, we just implement it! */ +#define nsdpoll_ptcpCURR_IF_VERSION nsdCURR_IF_VERSION + +/* prototypes */ +PROTOTYPEObj(nsdpoll_ptcp); + +#endif /* #ifndef INCLUDED_NSDPOLL_PTCP_H */ diff --git a/runtime/nsdsel_ptcp.c b/runtime/nsdsel_ptcp.c index 41b85e0c..e2cfca7c 100644 --- a/runtime/nsdsel_ptcp.c +++ b/runtime/nsdsel_ptcp.c @@ -36,6 +36,7 @@ #include "errmsg.h" #include "nsd_ptcp.h" #include "nsdsel_ptcp.h" +#include "unlimited_select.h" /* static data */ DEFobjStaticHelpers @@ -47,14 +48,23 @@ DEFobjCurrIf(glbl) */ BEGINobjConstruct(nsdsel_ptcp) /* be sure to specify the object type also in END macro! */ pThis->maxfds = 0; +#ifdef USE_UNLIMITED_SELECT + pThis->pReadfds = calloc(1, glbl.GetFdSetSize()); + pThis->pWritefds = calloc(1, glbl.GetFdSetSize()); +#else FD_ZERO(&pThis->readfds); FD_ZERO(&pThis->writefds); +#endif ENDobjConstruct(nsdsel_ptcp) /* destructor for the nsdsel_ptcp object */ BEGINobjDestruct(nsdsel_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(nsdsel_ptcp) +#ifdef USE_UNLIMITED_SELECT + freeFdSet(pThis->pReadfds); + freeFdSet(pThis->pWritefds); +#endif ENDobjDestruct(nsdsel_ptcp) @@ -65,20 +75,27 @@ Add(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pSock, nsd_ptcp); ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); switch(waitOp) { case NSDSEL_RD: - FD_SET(pSock->sock, &pThis->readfds); + FD_SET(pSock->sock, pReadfds); break; case NSDSEL_WR: - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - FD_SET(pSock->sock, &pThis->readfds); - FD_SET(pSock->sock, &pThis->writefds); + FD_SET(pSock->sock, pReadfds); + FD_SET(pSock->sock, pWritefds); break; } @@ -98,6 +115,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) DEFiRet; int i; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); assert(piNumReady != NULL); @@ -106,13 +130,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) // TODO: name in dbgprintf! dbgprintf("--------<NSDSEL_PTCP> calling select, active fds (max %d): ", pThis->maxfds); for(i = 0; i <= pThis->maxfds; ++i) - if(FD_ISSET(i, &pThis->readfds) || FD_ISSET(i, &pThis->writefds)) + if(FD_ISSET(i, pReadfds) || FD_ISSET(i, pWritefds)) dbgprintf("%d ", i); dbgprintf("\n"); } /* now do the select */ - *piNumReady = select(pThis->maxfds+1, &pThis->readfds, &pThis->writefds, NULL, NULL); + *piNumReady = select(pThis->maxfds+1, pReadfds, pWritefds, NULL, NULL); RETiRet; } @@ -125,6 +149,13 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) DEFiRet; nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel; nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds = pThis->pReadfds; + fd_set *pWritefds = pThis->pWritefds; +#else + fd_set *pReadfds = &pThis->readfds; + fd_set *pWritefds = &pThis->writefds; +#endif ISOBJ_TYPE_assert(pThis, nsdsel_ptcp); ISOBJ_TYPE_assert(pSock, nsd_ptcp); @@ -132,14 +163,14 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) switch(waitOp) { case NSDSEL_RD: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds); break; case NSDSEL_WR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pWritefds); break; case NSDSEL_RDWR: - *pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds) - | FD_ISSET(pSock->sock, &pThis->writefds); + *pbIsReady = FD_ISSET(pSock->sock, pReadfds) + | FD_ISSET(pSock->sock, pWritefds); break; } diff --git a/runtime/nsdsel_ptcp.h b/runtime/nsdsel_ptcp.h index 6c0c7fa7..f9ec8210 100644 --- a/runtime/nsdsel_ptcp.h +++ b/runtime/nsdsel_ptcp.h @@ -31,8 +31,13 @@ typedef nsdsel_if_t nsdsel_ptcp_if_t; /* we just *implement* this interface */ struct nsdsel_ptcp_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ int maxfds; +#ifdef USE_UNLIMITED_SELECT + fd_set *pReadfds; + fd_set *pWritefds; +#else fd_set readfds; fd_set writefds; +#endif }; /* interface is defined in nsd.h, we just implement it! */ diff --git a/runtime/nspoll.c b/runtime/nspoll.c new file mode 100644 index 00000000..f287cd4e --- /dev/null +++ b/runtime/nspoll.c @@ -0,0 +1,198 @@ +/* nspoll.c + * + * This is an io waiter interface utilizing the much-more-efficient poll/epoll API. + * Note that it may not always be available for a given driver. If so, that is reported + * back to the upper peer which then should consult a nssel-based io waiter. + * + * Work on this module begun 2009-11-18 by Rainer Gerhards. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include "rsyslog.h" +#include "obj.h" +#include "module-template.h" +#include "netstrm.h" +#include "nspoll.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) + + +/* load our low-level driver. This must be done before any + * driver-specific functions (allmost all...) can be carried + * out. Note that the driver's .ifIsLoaded is correctly + * initialized by calloc() and we depend on that. Please note that + * we do some name-mangeling. We know that each nsd driver also needs + * a nspoll driver. So we simply append "sel" to the nsd driver name: This, + * of course, means that the driver name must match these rules, but that + * shouldn't be a real problem. + * WARNING: this code is mostly identical to similar code in + * netstrms.c - TODO: abstract it and move it to some common place. + * rgerhards, 2008-04-28 + */ +static rsRetVal +loadDrvr(nspoll_t *pThis) +{ + DEFiRet; + uchar *pBaseDrvrName; + uchar szDrvrName[48]; /* 48 shall be large enough */ + + pBaseDrvrName = pThis->pBaseDrvrName; + if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdpoll_%s", pBaseDrvrName) == sizeof(szDrvrName)) + ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); + CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); + + pThis->Drvr.ifVersion = nsdCURR_IF_VERSION; + /* The pDrvrName+2 below is a hack to obtain the object name. It + * safes us to have yet another variable with the name without "lm" in + * front of it. If we change the module load interface, we may re-think + * about this hack, but for the time being it is efficient and clean + * enough. -- rgerhards, 2008-04-18 + */ +RUNLOG_VAR("%s", szDrvrName+2); + CHKiRet(obj.UseObj(__FILE__, szDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr)); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pDrvrName != NULL) + free(pThis->pDrvrName); + pThis->pDrvrName = NULL; + } + RETiRet; +} + + +/* Standard-Constructor */ +BEGINobjConstruct(nspoll) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(nspoll) + + +/* destructor for the nspoll object */ +BEGINobjDestruct(nspoll) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nspoll) + if(pThis->pDrvrData != NULL) + pThis->Drvr.Destruct(&pThis->pDrvrData); + + /* and now we must release our driver, if we got one. We use the presence of + * a driver name string as load indicator (because we also need that string + * to release the driver + */ + if(pThis->pDrvrName != NULL) { + obj.ReleaseObj(__FILE__, pThis->pDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr); + free(pThis->pDrvrName); + } +ENDobjDestruct(nspoll) + + +/* ConstructionFinalizer */ +static rsRetVal +ConstructFinalize(nspoll_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); +RUNLOG_STR("trying to load epoll driver\n"); + CHKiRet(loadDrvr(pThis)); + CHKiRet(pThis->Drvr.Construct(&pThis->pDrvrData)); +finalize_it: +dbgprintf("XXX: done trying to load epoll driver, state %d\n", iRet); + RETiRet; +} + + +/* Carries out the actual wait (all done in lower layers) + */ +static rsRetVal +Wait(nspoll_t *pThis, int timeout, int *idRdy, void **ppUsr) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + assert(idRdy != NULL); + iRet = pThis->Drvr.Wait(pThis->pDrvrData, timeout, idRdy, ppUsr); + RETiRet; +} + + +/* semantics like the epoll_ctl() function, does the same thing. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Ctl(nspoll_t *pThis, netstrm_t *pStrm, int id, void *pUsr, int mode, int op) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + iRet = pThis->Drvr.Ctl(pThis->pDrvrData, pStrm->pDrvrData, id, pUsr, mode, op); + RETiRet; +} + + +/* queryInterface function */ +BEGINobjQueryInterface(nspoll) +CODESTARTobjQueryInterface(nspoll) + if(pIf->ifVersion != nspollCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = nspollConstruct; + pIf->ConstructFinalize = ConstructFinalize; + pIf->Destruct = nspollDestruct; + pIf->Wait = Wait; + pIf->Ctl = Ctl; +finalize_it: +ENDobjQueryInterface(nspoll) + + +/* exit our class + */ +BEGINObjClassExit(nspoll, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nspoll) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(nspoll) + + +/* Initialize the nspoll class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nspoll, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + DBGPRINTF("doing nspollClassInit\n"); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nspoll) +/* vi:set ai: + */ diff --git a/runtime/nspoll.h b/runtime/nspoll.h new file mode 100644 index 00000000..a77759c0 --- /dev/null +++ b/runtime/nspoll.h @@ -0,0 +1,65 @@ +/* Definitions for the nspoll io activity waiter + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSPOLL_H +#define INCLUDED_NSPOLL_H + +#include "netstrms.h" + +/* some operations to be portable when we do not have epoll() available */ +#define NSDPOLL_ADD 1 +#define NSDPOLL_DEL 2 + +/* and some mode specifiers for waiting on input/output */ +#define NSDPOLL_IN 1 /* EPOLLIN */ +#define NSDPOLL_OUT 2 /* EPOLLOUT */ +/* next is 4, 8, 16, ... - must be bit values, as they are ored! */ + +/* the nspoll object */ +struct nspoll_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + nsd_t *pDrvrData; /**< the driver's data elements */ + uchar *pBaseDrvrName; /**< nsd base driver name to use, or NULL if system default */ + uchar *pDrvrName; /**< full base driver name (set when driver is loaded) */ + nsdpoll_if_t Drvr; /**< our stream driver */ +}; + + +/* interface */ +BEGINinterface(nspoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nspoll_t **ppThis); + rsRetVal (*ConstructFinalize)(nspoll_t *pThis); + rsRetVal (*Destruct)(nspoll_t **ppThis); + rsRetVal (*Wait)(nspoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); + rsRetVal (*Ctl)(nspoll_t *pNsdpoll, netstrm_t *pStrm, int id, void *pUsr, int mode, int op); + rsRetVal (*IsEPollSupported)(void); /* static method */ +ENDinterface(nspoll) +#define nspollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + +/* prototypes */ +PROTOTYPEObj(nspoll); + +/* the name of our library binary */ +#define LM_NSPOLL_FILENAME LM_NETSTRMS_FILENAME + +#endif /* #ifndef INCLUDED_NSPOLL_H */ diff --git a/runtime/nssel.c b/runtime/nssel.c index d11d5fe1..7c5be3a9 100644 --- a/runtime/nssel.c +++ b/runtime/nssel.c @@ -219,6 +219,7 @@ ENDObjClassExit(nssel) */ BEGINObjClassInit(nssel, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + DBGPRINTF("doing nsselClassInit\n"); CHKiRet(objUse(glbl, CORE_COMPONENT)); /* set our own handlers */ diff --git a/runtime/parser.c b/runtime/parser.c index 38f72986..ca31b35d 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -60,6 +60,8 @@ DEFobjCurrIf(ruleset) /* config data */ static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */ +static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ /* This is the list of all parsers known to us. @@ -308,7 +310,7 @@ SanitizeMsg(msg_t *pMsg) size_t iDst; size_t iMaxLine; size_t maxDest; - bool bUpdatedLen = FALSE; + sbool bUpdatedLen = FALSE; uchar szSanBuf[32*1024]; /* buffer used for sanitizing a string */ assert(pMsg != NULL); @@ -339,6 +341,11 @@ SanitizeMsg(msg_t *pMsg) * needs sanitation than to do the sanitation in any case. So we first do * this and terminate when it is not needed - which is expectedly the case * for the vast majority of messages. -- rgerhards, 2009-06-15 + * Note that we do NOT check here if tab characters are to be escaped or + * not. I expect this functionality to be seldomly used and thus I do not + * like to pay the performance penalty. So the penalty is only with those + * that actually use it, because we may call the sanitizer without actual + * need below (but it then still will work perfectly well!). -- rgerhards, 2009-11-27 */ int bNeedSanitize = 0; for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { @@ -347,6 +354,9 @@ SanitizeMsg(msg_t *pMsg) bNeedSanitize = 1; break; } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + bNeedSanitize = 1; + break; } } @@ -367,7 +377,7 @@ SanitizeMsg(msg_t *pMsg) CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ - if(iscntrl((int) pszMsg[iSrc])) { + if(iscntrl((int) pszMsg[iSrc]) && (pszMsg[iSrc] != '\t' || bEscapeTab)) { /* note: \0 must always be escaped, the rest of the code currently * can not handle it! -- rgerhards, 2009-08-26 */ @@ -381,6 +391,14 @@ SanitizeMsg(msg_t *pMsg) pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + /* In this case, we also do the conversion. Note that this most + * probably breaks European languages. -- rgerhards, 2010-01-27 + */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); } else { pDst[iDst++] = pszMsg[iSrc]; } @@ -446,8 +464,8 @@ ParseMsg(msg_t *pMsg) rsRetVal localRet; parserList_t *pParserList; parser_t *pParser; - bool bIsSanitized; - bool bPRIisParsed; + sbool bIsSanitized; + sbool bPRIisParsed; static int iErrMsgRateLimiter = 0; DEFiRet; @@ -462,7 +480,8 @@ ParseMsg(msg_t *pMsg) * (and that functionality is too important for debugging to drop it...). */ DBGPRINTF("msg parser: flags %x, from '%s', msg '%.50s'\n", pMsg->msgFlags, - getRcvFrom(pMsg), pMsg->pszRawMsg); + (pMsg->msgFlags & NEEDS_DNSRESOL) ? UCHAR_CONSTANT("~NOTRESOLVED~") : getRcvFrom(pMsg), + pMsg->pszRawMsg); /* we now need to go through our list of parsers and see which one is capable of * parsing the message. Note that the first parser that requires message sanitization @@ -618,6 +637,8 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus { cCCEscapeChar = '#'; bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bEscape8BitChars = 0; /* default is to escape control characters */ + bEscapeTab = 1; /* default is to escape control characters */ bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ return RS_RET_OK; @@ -669,6 +690,8 @@ BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); InitParserList(&pParsLstRoot); diff --git a/runtime/parser.h b/runtime/parser.h index c4f63021..bdd572cb 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -38,8 +38,8 @@ struct parser_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ uchar *pName; /* name of this parser */ modInfo_t *pModule; /* pointer to parser's module */ - bool bDoSanitazion; /* do standard message sanitazion before calling parser? */ - bool bDoPRIParsing; /* do standard PRI parsing before calling parser? */ + sbool bDoSanitazion; /* do standard message sanitazion before calling parser? */ + sbool bDoPRIParsing; /* do standard PRI parsing before calling parser? */ }; /* interfaces */ diff --git a/runtime/queue.h b/runtime/queue.h index 93573dae..38c0d491 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -60,9 +60,9 @@ struct queue_s { queueType_t qType; int nLogDeq; /* number of elements currently logically dequeued */ int bShutdownImmediate; /* should all workers cease processing messages? */ - bool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ - bool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ - bool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ + sbool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ + sbool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ + sbool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ int iQueueSize; /* Current number of elements in the queue */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ @@ -73,14 +73,14 @@ struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ - bool bSyncQueueFiles;/* if working with files, sync them after each write? */ + sbool bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ int iFullDlyMrk; /* if the queue is above this mark, FULL_DELAYable message are put on hold */ int iLightDlyMrk; /* if the queue is above this mark, LIGHT_DELAYable message are put on hold */ int iDiscardSeverity;/* messages of this severity above are discarded on too-full queue */ - bool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ + sbool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ int toQShutdown; /* timeout for regular queue shutdown in ms */ int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index e5e89b1f..a75d2bc0 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -107,6 +107,7 @@ typedef struct NetAddr netAddr_t; typedef struct netstrms_s netstrms_t; typedef struct netstrm_s netstrm_t; typedef struct nssel_s nssel_t; +typedef struct nspoll_s nspoll_t; typedef enum nsdsel_waitOp_e nsdsel_waitOp_t; typedef struct nsd_ptcp_s nsd_ptcp_t; typedef struct nsd_gtls_s nsd_gtls_t; @@ -114,9 +115,11 @@ typedef struct nsd_gsspi_s nsd_gsspi_t; typedef struct nsd_nss_s nsd_nss_t; typedef struct nsdsel_ptcp_s nsdsel_ptcp_t; typedef struct nsdsel_gtls_s nsdsel_gtls_t; +typedef struct nsdpoll_ptcp_s nsdpoll_ptcp_t; typedef struct wti_s wti_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; +typedef obj_t nsdpoll_t; typedef struct msg msg_t; typedef struct queue_s qqueue_t; typedef struct prop_s prop_t; @@ -148,7 +151,9 @@ typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif -typedef char bool; /* I intentionally use char, to keep it slim so that many fit into the CPU cache! */ +typedef struct epoll_event epoll_event_t; + +typedef char sbool; /* (small bool) I intentionally use char, to keep it slim so that many fit into the CPU cache! */ /* settings for flow control * TODO: is there a better place for them? -- rgerhards, 2008-03-14 @@ -409,6 +414,11 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */ RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */ RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */ + RS_RET_EINTR = -2161, /**< EINTR occured during a system call (not necessarily an error) */ + RS_RET_ERR_EPOLL = -2162, /**< epoll() returned with an unexpected error code */ + RS_RET_ERR_EPOLL_CTL = -2163, /**< epol_ctll() returned with an unexpected error code */ + RS_RET_TIMEOUT = -2164, /**< timeout occured during operation */ + RS_RET_RCV_ERR = -2165, /**< error occured during socket rcv operation */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/rule.h b/runtime/rule.h index 99ac44e7..7b607637 100644 --- a/runtime/rule.h +++ b/runtime/rule.h @@ -47,7 +47,7 @@ struct rule_s { fiop_t operation; regex_t *regex_cache; /* cache for compiled REs, if such are used */ cstr_t *pCSCompValue; /* value to "compare" against */ - bool isNegated; + sbool isNegated; propid_t propID; /* ID of the requested property */ } prop; expr_t *f_expr; /* expression object */ diff --git a/runtime/srutils.c b/runtime/srutils.c index 7ddc3ba2..2bed624e 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -46,6 +46,9 @@ #include "srUtils.h" #include "obj.h" +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif /* here we host some syslog specific names. There currently is no better place * to do it, but over here is also not ideal... -- rgerhards, 2008-02-14 @@ -372,10 +375,22 @@ int getNumberDigits(long lNum) rsRetVal timeoutComp(struct timespec *pt, long iTimeout) { +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ + +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, pt); +# else + gettimeofday(&tv, NULL); + pt->tv_sec = tv.tv_sec; + pt->tv_nsec = tv.tv_usec * 1000; +# endif pt->tv_sec += iTimeout / 1000; pt->tv_nsec += (iTimeout % 1000) * 1000000; /* think INTEGER arithmetic! */ if(pt->tv_nsec > 999999999) { /* overrun? */ @@ -397,11 +412,21 @@ timeoutVal(struct timespec *pt) { struct timespec t; long iTimeout; - BEGINfunc +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif iTimeout = (pt->tv_nsec - t.tv_nsec) / 1000000; iTimeout += (pt->tv_sec - t.tv_sec) * 1000; diff --git a/runtime/stream.c b/runtime/stream.c index 81f8e89b..f565dc90 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -902,7 +902,7 @@ asyncWriterThread(void *pPtr) { int iDeq; struct timespec t; - bool bTimedOut = 0; + sbool bTimedOut = 0; strm_t *pThis = (strm_t*) pPtr; ISOBJ_TYPE_assert(pThis, strm); @@ -1068,7 +1068,7 @@ doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { z_stream zstrm; int zRet; /* zlib return state */ - bool bzInitDone = FALSE; + sbool bzInitDone = FALSE; DEFiRet; assert(pThis != NULL); assert(pBuf != NULL); diff --git a/runtime/stream.h b/runtime/stream.h index 1dc92807..5e3324c5 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -102,12 +102,12 @@ typedef struct strm_s { int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ - bool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ + sbool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ - bool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ - bool bSync; /* sync this file after every write? */ + sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ + sbool bSync; /* sync this file after every write? */ size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; @@ -118,14 +118,14 @@ typedef struct strm_s { size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ - bool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ - bool bInClose; /* used to break "deadly close loops", tells us we are already inside a close */ + sbool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + sbool bInClose; /* used to break "deadly close loops", tells us we are already inside a close */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; /* support for async flush procesing */ - bool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ - bool bStopWriter; /* shall writer thread terminate? */ - bool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ + sbool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ + sbool bStopWriter; /* shall writer thread terminate? */ + sbool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ int iFlushInterval; /* flush in which interval - 0, no flushing */ apc_id_t apcID; /* id of current Apc request (used for cancelling) */ pthread_mutex_t mut;/* mutex for flush in async mode */ @@ -144,7 +144,7 @@ typedef struct strm_s { /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ - bool bIsTTY; /* is this a tty file? */ + sbool bIsTTY; /* is this a tty file? */ } strm_t; diff --git a/runtime/unlimited_select.h b/runtime/unlimited_select.h new file mode 100644 index 00000000..32dadc03 --- /dev/null +++ b/runtime/unlimited_select.h @@ -0,0 +1,45 @@ +/* unlimited_select.h + * Tweak the macros for accessing fd_set so that the select() syscall + * won't be limited to a particular number of file descriptors. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +#ifndef UNLIMITED_SELECT_H_INCLUDED + +#include <string.h> +#include <stdlib.h> +#include <sys/select.h> +#include "glbl.h" + +#ifdef USE_UNLIMITED_SELECT +# undef FD_ZERO +# define FD_ZERO(set) memset((set), 0, glbl.GetFdSetSize()); +#endif + +#ifdef USE_UNLIMITED_SELECT +void freeFdSet(fd_set *p) { + free(p); +} +#else +# define freeFdSet(x) +#endif + +#endif /* #ifndef UNLIMITED_SELECT_H_INCLUDED */ diff --git a/runtime/wti.c b/runtime/wti.c index 288670b6..14964fb0 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -79,7 +79,7 @@ wtiGetDbgHdr(wti_t *pThis) /* return the current worker processing state. For the sake of * simplicity, we do not use the iRet interface. -- rgerhards, 2009-07-17 */ -bool +sbool wtiGetState(wti_t *pThis) { return ATOMIC_FETCH_32BIT(pThis->bIsRunning); @@ -102,7 +102,7 @@ wtiSetAlwaysRunning(wti_t *pThis) * is inside wti). -- rgerhards, 2009-07-17 */ rsRetVal -wtiSetState(wti_t *pThis, bool bNewVal) +wtiSetState(wti_t *pThis, sbool bNewVal) { ISOBJ_TYPE_assert(pThis, wti); if(bNewVal) diff --git a/runtime/wti.h b/runtime/wti.h index f466a053..e587c69e 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -35,7 +35,7 @@ struct wti_s { BEGINobjInstance; pthread_t thrdID; /* thread ID */ int bIsRunning; /* is this thread currently running? (must be int for atomic op!) */ - bool bAlwaysRunning; /* should this thread always run? */ + sbool bAlwaysRunning; /* should this thread always run? */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ uchar *pszDbgHdr; /* header string for debug messages */ @@ -50,8 +50,8 @@ rsRetVal wtiWorker(wti_t *pThis); rsRetVal wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg); rsRetVal wtiCancelThrd(wti_t *pThis); rsRetVal wtiSetAlwaysRunning(wti_t *pThis); -rsRetVal wtiSetState(wti_t *pThis, bool bNew); -bool wtiGetState(wti_t *pThis); +rsRetVal wtiSetState(wti_t *pThis, sbool bNew); +sbool wtiGetState(wti_t *pThis); PROTOTYPEObjClassInit(wti); PROTOTYPEpropSetMeth(wti, pszDbgHdr, uchar*); PROTOTYPEpropSetMeth(wti, pWtp, wtp_t*); diff --git a/runtime/wtp.c b/runtime/wtp.c index 060e6627..ab7ca4bb 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -442,7 +442,6 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) CHKiRet(wtpStartWrkr(pThis)); } } else { -dbgprintf("YYY: wtpAdviseMaxWorkers, sufficient workers, just doing adivse signal cond busy\n"); pthread_cond_signal(pThis->pcondBusy); } diff --git a/tcps_sess.c b/tcps_sess.c index a3cd2f30..69b40ad0 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -407,7 +407,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt */ } - if(( (c == '\n') + if(( ((c == '\n') && !pThis->pSrv->bDisableLFDelim) || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); @@ -15,12 +15,9 @@ * callbacks before the code is run. The tcpsrv then calls back * into the specific input modules at the appropriate time. * - * NOTE: read comments in module-template.h to understand how this file - * works! - * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -68,6 +65,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "errmsg.h" #include "ruleset.h" #include "unicode-helper.h" @@ -89,6 +87,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) +DEFobjCurrIf(nspoll) DEFobjCurrIf(prop) @@ -238,11 +237,13 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) if(pThis->pSessions != NULL) { /* close all TCP connections! */ - i = TCPSessGetNxtSess(pThis, -1); - while(i != -1) { - tcps_sess.Destruct(&pThis->pSessions[i]); - /* now get next... */ - i = TCPSessGetNxtSess(pThis, i); + if(!pThis->bUsingEPoll) { + i = TCPSessGetNxtSess(pThis, -1); + while(i != -1) { + tcps_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = TCPSessGetNxtSess(pThis, i); + } } /* we are done with the session table - so get rid of it... */ @@ -438,7 +439,8 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, } *ppSess = pSess; - pThis->pSessions[iSess] = pSess; + if(!pThis->bUsingEPoll) + pThis->pSessions[iSess] = pSess; pSess = NULL; /* this is now also handed over */ finalize_it: @@ -465,11 +467,29 @@ RunCancelCleanup(void *arg) } +/* helper to close a session. Takes status of poll vs. select into consideration. + * rgerhards, 2009-11-25 + */ +static inline rsRetVal +closeSess(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) { + DEFiRet; + if(pPoll != NULL) { + CHKiRet(nspoll.Ctl(pPoll, (*ppSess)->pStrm, 0, *ppSess, NSDPOLL_IN, NSDPOLL_DEL)); + } + pThis->pOnRegularClose(*ppSess); + tcps_sess.Destruct(ppSess); +finalize_it: + RETiRet; +} + + /* process a receive request on one of the streams + * If pPoll is non-NULL, we have a netstream in epoll mode, which means we need + * to remove any descriptor we close from the epoll set. * rgerhards, 2009-07-020 */ static rsRetVal -doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) +doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) { char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ ssize_t iRcvd; @@ -478,7 +498,6 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) ISOBJ_TYPE_assert(pThis, tcpsrv); DBGPRINTF("netstream %p with new data\n", (*ppSess)->pStrm); - /* Receive message */ iRet = pThis->pRcvData(*ppSess, buf, sizeof(buf), &iRcvd); switch(iRet) { @@ -491,8 +510,7 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "Netstream session %p closed by remote peer %s.\n", (*ppSess)->pStrm, pszPeer); } - pThis->pOnRegularClose(*ppSess); - tcps_sess.Destruct(ppSess); + CHKiRet(closeSess(pThis, ppSess, pPoll)); break; case RS_RET_RETRY: /* we simply ignore retry - this is not an error, but we also have not received anything */ @@ -505,26 +523,29 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess) */ errmsg.LogError(0, localRet, "Tearing down TCP Session - see " "previous messages for reason(s)\n"); - pThis->pOnErrClose(*ppSess); - tcps_sess.Destruct(ppSess); + CHKiRet(closeSess(pThis, ppSess, pPoll)); } break; default: errno = 0; errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", (*ppSess)->pStrm); - pThis->pOnErrClose(*ppSess); - tcps_sess.Destruct(ppSess); + CHKiRet(closeSess(pThis, ppSess, pPoll)); break; } + +finalize_it: RETiRet; } -/* This function is called to gather input. */ +/* This function is called to gather input. + * This variant here is only used if we need to work with a netstream driver + * that does not support epoll(). + */ #pragma GCC diagnostic ignored "-Wempty-body" -static rsRetVal -Run(tcpsrv_t *pThis) +static inline rsRetVal +RunSelect(tcpsrv_t *pThis) { DEFiRet; int nfds; @@ -532,7 +553,7 @@ Run(tcpsrv_t *pThis) int iTCPSess; int bIsReady; tcps_sess_t *pNewSess; - nssel_t *pSel; + nssel_t *pSel = NULL; ISOBJ_TYPE_assert(pThis, tcpsrv); @@ -583,7 +604,7 @@ Run(tcpsrv_t *pThis) ABORT_FINALIZE(RS_RET_FORCE_TERM); CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - doReceive(pThis, &pThis->pSessions[iTCPSess]); + doReceive(pThis, &pThis->pSessions[iTCPSess], NULL); --nfds; /* indicate we have processed one */ } iTCPSess = TCPSessGetNxtSess(pThis, iTCPSess); @@ -606,11 +627,93 @@ finalize_it: /* this is a very special case - this time only we do not exit the #pragma GCC diagnostic warning "-Wempty-body" +/* This function is called to gather input. It tries doing that via the epoll() + * interface. If the driver does not support that, it falls back to calling its + * select() equivalent. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Run(tcpsrv_t *pThis) +{ + DEFiRet; + int i; + tcps_sess_t *pNewSess; + nspoll_t *pPoll = NULL; + void *pUsr; + rsRetVal localRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerhards, 20080-04-24 + */ + if((localRet = nspoll.Construct(&pPoll)) == RS_RET_OK) { + // TODO: set driver + localRet = nspoll.ConstructFinalize(pPoll); + } + if(localRet != RS_RET_OK) { + /* fall back to select */ + dbgprintf("tcpsrv could not use epoll() interface, iRet=%d, using select()\n", localRet); + iRet = RunSelect(pThis); + FINALIZE; + } + + dbgprintf("tcpsrv uses epoll() interface, nsdpol driver found\n"); + + /* flag that we are in epoll mode */ + pThis->bUsingEPoll = TRUE; + + /* Add the TCP listen sockets to the list of sockets to monitor */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + dbgprintf("Trying to add listener %d, pUsr=%p\n", i, pThis->ppLstn); + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_ADD)); + dbgprintf("Added listener %d\n", i); + } + + while(1) { + localRet = nspoll.Wait(pPoll, -1, &i, &pUsr); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + + /* check if we need to ignore the i/o ready state. We do this if we got an invalid + * return state. Validly, this can happen for RS_RET_EINTR, for other cases it may + * not be the right thing, but what is the right thing is really hard at this point... + */ + if(localRet != RS_RET_OK) + continue; + + dbgprintf("poll returned with i %d, pUsr %p\n", i, pUsr); + + if(pUsr == pThis->ppLstn) { + DBGPRINTF("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + CHKiRet(nspoll.Ctl(pPoll, pNewSess->pStrm, 0, pNewSess, NSDPOLL_IN, NSDPOLL_ADD)); + DBGPRINTF("New session created with NSD %p.\n", pNewSess); + } else { + pNewSess = (tcps_sess_t*) pUsr; + doReceive(pThis, &pNewSess, pPoll); + } + } + + /* remove the tcp listen sockets from the epoll set */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_DEL)); + } + +finalize_it: + if(pPoll != NULL) + nspoll.Destruct(&pPoll); + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ pThis->iSessMax = TCPSESS_MAX_DEFAULT; pThis->iLstnMax = TCPLSTN_MAX_DEFAULT; pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + pThis->bDisableLFDelim = 0; pThis->OnMsgReceive = NULL; ENDobjConstruct(tcpsrv) @@ -767,6 +870,18 @@ SetOnMsgReceive(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, } +/* set enable/disable standard LF frame delimiter (use with care!) + * -- rgerhards, 2010-01-03 + */ +static rsRetVal +SetbDisableLFDelim(tcpsrv_t *pThis, int bVal) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->bDisableLFDelim = bVal; + RETiRet; +} + /* Set additional framing to use (if any) -- rgerhards, 2008-12-10 */ static rsRetVal @@ -908,7 +1023,6 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->ConstructFinalize = tcpsrvConstructFinalize; pIf->Destruct = tcpsrvDestruct; - //pIf->SessAccept = SessAccept; pIf->configureTCPListen = configureTCPListen; pIf->create_tcp_socket = create_tcp_socket; pIf->Run = Run; @@ -916,6 +1030,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetUsrP = SetUsrP; pIf->SetInputName = SetInputName; pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; + pIf->SetbDisableLFDelim = SetbDisableLFDelim; pIf->SetSessMax = SetSessMax; pIf->SetLstnMax = SetLstnMax; pIf->SetDrvrMode = SetDrvrMode; @@ -969,6 +1084,7 @@ BEGINObjClassInit(tcpsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(nspoll, DONT_LOAD_LIB)); CHKiRet(objUse(tcps_sess, DONT_LOAD_LIB)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); @@ -54,7 +54,8 @@ struct tcpsrv_s { uchar *pszInputName; /**< value to be used as input name */ ruleset_t *pRuleset; /**< ruleset to bind to */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ - bool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ + sbool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ + sbool bUsingEPoll; /**< are we in epoll mode (means we do not need to keep track of sessions!) */ int iLstnCurr; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ @@ -63,6 +64,7 @@ struct tcpsrv_s { tcpLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ int addtlFrameDelim; /**< additional frame delimiter for plain TCP syslog framing (e.g. to handle NetScreen) */ + int bDisableLFDelim; /**< if 1, standard LF frame delimiter is disabled (*very dangerous*) */ tcps_sess_t **pSessions;/**< array of all of our sessions */ void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ /* callbacks */ @@ -114,11 +116,13 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ /* added v6 */ rsRetVal (*SetOnMsgReceive)(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); /* 2009-05-24 */ rsRetVal (*SetRuleset)(tcpsrv_t *pThis, ruleset_t*); /* 2009-06-12 */ - /* added v7 */ + /* added v7 (accidently named v8!) */ rsRetVal (*SetLstnMax)(tcpsrv_t *pThis, int iMaxLstn); /* 2009-08-17 */ rsRetVal (*SetNotificationOnRemoteClose)(tcpsrv_t *pThis, int bNewVal); /* 2009-10-01 */ + /* added v9 -- rgerhards, 2010-03-01 */ + rsRetVal (*SetbDisableLFDelim)(tcpsrv_t*, int); ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 8 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 diff --git a/tests/Makefile.am b/tests/Makefile.am index 61fc808e..63dba939 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -20,6 +20,8 @@ TESTS = $(TESTRUNS) cfg.sh \ queue-persist.sh \ pipeaction.sh \ execonlyonce.sh \ + dircreate_dflt.sh \ + dircreate_off.sh \ queue-persist.sh if ENABLE_OMUDPSPOOF @@ -37,6 +39,8 @@ TESTS += omod-if-array.sh \ threadingmqaq.sh \ discard.sh \ badqi.sh \ + tabescape_dflt.sh \ + tabescape_off.sh \ fieldtest.sh endif @@ -93,12 +97,17 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/master.tspgsql \ testsuites/subsecond.conf \ testsuites/master.subsecond \ + testsuites/parse_8bit_escape.conf \ + testsuites/8bit.parse_8bit_escape \ testsuites/parse1.conf \ testsuites/field1.conf \ testsuites/1.parse1 \ testsuites/2.parse1 \ testsuites/3.parse1 \ testsuites/4.parse1 \ + testsuites/mark.parse1 \ + testsuites/8bit.parse1 \ + testsuites/empty.parse1 \ testsuites/oversizeTag-1.parse1 \ testsuites/weird.parse1 \ testsuites/date1.parse1 \ @@ -199,6 +208,16 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ execonlyonce.sh \ testsuites/execonlyonce.conf \ testsuites/execonlyonce.data \ + tabescape_dflt.sh \ + testsuites/tabescape_dflt.conf \ + testsuites/1.tabescape_dflt \ + tabescape_off.sh \ + testsuites/tabescape_off.conf \ + testsuites/1.tabescape_off \ + dircreate_dflt.sh \ + testsuites/dircreate_dflt.conf \ + dircreate_off.sh \ + testsuites/dircreate_off.conf \ DiagTalker.java \ cfg.sh diff --git a/tests/diag.sh b/tests/diag.sh index 98228b12..1f410645 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -18,14 +18,14 @@ case $1 in rm -f rsyslogd.started work-*.conf rm -f rsyslogd2.started work-*.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files - rm -rf test-spool + rm -rf test-spool test-logdir rm -f core.* vgcore.* mkdir test-spool ;; 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf rm -f rsyslogd2.started diag-common2.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files - rm -rf test-spool + rm -rf test-spool test-logdir echo ------------------------------------------------------------------------------- ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use diff --git a/tests/dircreate_dflt.sh b/tests/dircreate_dflt.sh new file mode 100755 index 00000000..71a671f3 --- /dev/null +++ b/tests/dircreate_dflt.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_dflt_dflt.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_dflt.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ ! -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile not created!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/dircreate_off.sh b/tests/dircreate_off.sh new file mode 100755 index 00000000..92fdee01 --- /dev/null +++ b/tests/dircreate_off.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_off_off.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_off.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile WAS created where not permitted to!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/parsertest.sh b/tests/parsertest.sh index fb392db3..06fcc8d6 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -4,6 +4,8 @@ source $srcdir/diag.sh nettester parse1 udp source $srcdir/diag.sh nettester parse1 tcp source $srcdir/diag.sh nettester parse2 udp source $srcdir/diag.sh nettester parse2 tcp +source $srcdir/diag.sh nettester parse_8bit_escape udp +source $srcdir/diag.sh nettester parse_8bit_escape tcp source $srcdir/diag.sh nettester parse3 udp source $srcdir/diag.sh nettester parse3 tcp source $srcdir/diag.sh nettester parse_invld_regex udp @@ -14,6 +16,8 @@ source $srcdir/diag.sh nettester parse1 udp -4 source $srcdir/diag.sh nettester parse1 tcp -4 source $srcdir/diag.sh nettester parse2 udp -4 source $srcdir/diag.sh nettester parse2 tcp -4 +source $srcdir/diag.sh nettester parse_8bit_escape udp -4 +source $srcdir/diag.sh nettester parse_8bit_escape tcp -4 source $srcdir/diag.sh nettester parse3 udp -4 source $srcdir/diag.sh nettester parse3 tcp -4 source $srcdir/diag.sh nettester parse_invld_regex udp -4 diff --git a/tests/tabescape_dflt.sh b/tests/tabescape_dflt.sh new file mode 100755 index 00000000..d0e13ec9 --- /dev/null +++ b/tests/tabescape_dflt.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_dflt.sh\]: test for default tab escaping +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_dflt -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_dflt -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/tabescape_off.sh b/tests/tabescape_off.sh new file mode 100755 index 00000000..71ede7c0 --- /dev/null +++ b/tests/tabescape_off.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_off.sh\]: test for tab escaping off +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_off -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_off -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/testsuites/1.tabescape_dflt b/tests/testsuites/1.tabescape_dflt new file mode 100644 index 00000000..91444bd3 --- /dev/null +++ b/tests/testsuites/1.tabescape_dflt @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT#011after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.tabescape_off b/tests/testsuites/1.tabescape_off new file mode 100644 index 00000000..6a331c35 --- /dev/null +++ b/tests/testsuites/1.tabescape_off @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/8bit.parse1 b/tests/testsuites/8bit.parse1 new file mode 100644 index 00000000..90db6352 --- /dev/null +++ b/tests/testsuites/8bit.parse1 @@ -0,0 +1,2 @@ +<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü +6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: äöü diff --git a/tests/testsuites/8bit.parse_8bit_escape b/tests/testsuites/8bit.parse_8bit_escape new file mode 100644 index 00000000..b2f6335c --- /dev/null +++ b/tests/testsuites/8bit.parse_8bit_escape @@ -0,0 +1,2 @@ +<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü +6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: #303#244#303#266#303#274 diff --git a/tests/testsuites/dircreate_dflt.conf b/tests/testsuites/dircreate_dflt.conf new file mode 100644 index 00000000..9b9aadb8 --- /dev/null +++ b/tests/testsuites/dircreate_dflt.conf @@ -0,0 +1,11 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/dircreate_off.conf b/tests/testsuites/dircreate_off.conf new file mode 100644 index 00000000..28ccbd8c --- /dev/null +++ b/tests/testsuites/dircreate_off.conf @@ -0,0 +1,12 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$CreateDirs off +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/empty.parse1 b/tests/testsuites/empty.parse1 new file mode 100644 index 00000000..86a86986 --- /dev/null +++ b/tests/testsuites/empty.parse1 @@ -0,0 +1,3 @@ +<14>Jan 6 2009 15:22:26 localhost +14,user,info,Jan 6 15:22:26,localhost,,, +#Note: there is one space after localhost, but then \n! diff --git a/tests/testsuites/mark.parse1 b/tests/testsuites/mark.parse1 new file mode 100644 index 00000000..fff9ae6d --- /dev/null +++ b/tests/testsuites/mark.parse1 @@ -0,0 +1,7 @@ +#This is a malformed message, but one from real life. At least, +#it should be parsed as can be seen here. +<6>Feb 18 16:01:59 serverX -- MARK -- +6,kern,info,Feb 18 16:01:59,serverX,--,--, MARK -- +# and the next one as an extreme case (note the absence of PRI) +Feb 18 16:01:59 serverX -- MARK -- +13,user,notice,Feb 18 16:01:59,serverX,--,--, MARK -- diff --git a/tests/testsuites/parse_8bit_escape.conf b/tests/testsuites/parse_8bit_escape.conf new file mode 100644 index 00000000..0598f33f --- /dev/null +++ b/tests/testsuites/parse_8bit_escape.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off +$Escape8BitCharactersOnReceive on + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/tabescape_dflt.conf b/tests/testsuites/tabescape_dflt.conf new file mode 100644 index 00000000..b9d92a37 --- /dev/null +++ b/tests/testsuites/tabescape_dflt.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tabescape_off.conf b/tests/testsuites/tabescape_off.conf new file mode 100644 index 00000000..c1eca305 --- /dev/null +++ b/tests/testsuites/tabescape_off.conf @@ -0,0 +1,10 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +$EscapeControlCharacterTab off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt @@ -207,7 +207,7 @@ static void* thrdStarter(void *arg) * executing threads. It is added at the end of the list. * rgerhards, 2007-12-14 */ -rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), bool bNeedsCancel) +rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), sbool bNeedsCancel) { DEFiRet; thrdInfo_t *pThis; @@ -32,7 +32,7 @@ struct thrdInfo { rsRetVal (*pUsrThrdMain)(struct thrdInfo*); /* user thread main to be called in new thread */ rsRetVal (*pAfterRun)(struct thrdInfo*); /* cleanup function */ pthread_t thrdID; - bool bNeedsCancel; /* must input be terminated by pthread_cancel()? */ + sbool bNeedsCancel; /* must input be terminated by pthread_cancel()? */ }; /* prototypes */ @@ -40,7 +40,7 @@ rsRetVal thrdExit(void); rsRetVal thrdInit(void); rsRetVal thrdTerminate(thrdInfo_t *pThis); rsRetVal thrdTerminateAll(void); -rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), bool); +rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), sbool); /* macros (replace inline functions) */ diff --git a/tools/omfile.c b/tools/omfile.c index 2499680d..424dd70f 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -123,7 +123,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ -static bool bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */ +static sbool bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */ static int64 iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ uchar *pszFileDfltTplName = NULL; /* name of the default template to use */ @@ -139,7 +139,7 @@ typedef struct _instanceData { int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ - bool bForceChown; /* force chown() on existing files? */ + sbool bForceChown; /* force chown() on existing files? */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -158,7 +158,7 @@ typedef struct _instanceData { int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ int iFlushInterval; /* how fast flush buffer on inactivity? */ - bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ + sbool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; diff --git a/tools/syslogd.c b/tools/syslogd.c index b0a5b3ad..1ba3ef2b 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -574,7 +574,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) * permits us to process unmodified config files which otherwise contain a * supressor statement. */ - if(((Debug || NoFork) && bErrMsgToStderr) || iConfigVerify) { + if(((Debug == DEBUG_FULL || NoFork) && bErrMsgToStderr) || iConfigVerify) { if(LOG_PRI(pri) == LOG_ERR) fprintf(stderr, "rsyslogd: %s\n", msg); } @@ -591,6 +591,82 @@ finalize_it: RETiRet; } +/* check message against ACL set + * rgerhards, 2009-11-16 + */ +#if 0 +static inline rsRetVal +chkMsgAgainstACL() { + /* if we reach this point, we had a good receive and can process the packet received */ + /* check if we have a different sender than before, if so, we need to query some new values */ + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us + * syslog messages. If it isn't, we do not further + * process the message but log a warning (if we are + * configured to do this). + * rgerhards, 2005-09-26 + */ + *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", + (struct sockaddr *)&frominet, (char*)fromHostFQDN); + + if(!*pbIsPermitted) { + DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender %s discarded", + (char*)fromHost); + } + } + } + } +} +#endif + + +/* consumes a single messages - this function is primarily used to shuffle + * out some code from msgConsumer(). After this function, the message is + * (by definition!) considered committed. + * rgerhards, 2009-11-16 + */ +static inline rsRetVal +msgConsumeOne(msg_t *pMsg, prop_t **propFromHost, prop_t **propFromHostIP) { + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + int bIsPermitted; + DEFiRet; + + if((pMsg->msgFlags & NEEDS_ACLCHK_U) != 0) { + dbgprintf("msgConsumer: UDP ACL must be checked for message (hostname-based)\n"); + CHKiRet(net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP)); + bIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)pMsg->rcvFrom.pfrominet, (char*)fromHostFQDN, 1); + if(!bIsPermitted) { + DBGPRINTF("Message from '%s' discarded, not a permitted sender host\n", + fromHostFQDN); + ABORT_FINALIZE(RS_RET_ERR); + /* save some of the info we obtained */ + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), propFromHostIP)); + pMsg->msgFlags &= ~NEEDS_ACLCHK_U; + } + } + + if((pMsg->msgFlags & NEEDS_PARSING) != 0) + CHKiRet(parser.ParseMsg(pMsg)); + + ruleset.ProcessMsg(pMsg); +finalize_it: + RETiRet; +} + /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE @@ -602,26 +678,22 @@ static rsRetVal msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShutdownImmediate) { int i; - msg_t *pMsg; - rsRetVal localRet; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; DEFiRet; assert(pBatch != NULL); for(i = 0 ; i < pBatch->nElem && !*pbShutdownImmediate ; i++) { - pMsg = (msg_t*) pBatch->pElem[i].pUsrp; DBGPRINTF("msgConsumer processes msg %d/%d\n", i, pBatch->nElem); - if((pMsg->msgFlags & NEEDS_PARSING) != 0) { - localRet = parser.ParseMsg(pMsg); - if(localRet == RS_RET_OK) - ruleset.ProcessMsg(pMsg); - } else { - ruleset.ProcessMsg(pMsg); - } - /* if we reach this point, the message is considered committed (by definition!) */ + msgConsumeOne((msg_t*) pBatch->pElem[i].pUsrp, &propFromHost, &propFromHostIP); pBatch->pElem[i].state = BATCH_STATE_COMM; } + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); RETiRet; } @@ -910,9 +982,10 @@ static void doDie(int sig) static int iRetries = 0; /* debug aid */ dbgprintf(MSG1); if(Debug) + if(Debug == DEBUG_FULL) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { - if(Debug) + if(Debug == DEBUG_FULL) write(1, MSG2, sizeof(MSG2) - 1); abort(); } @@ -1091,6 +1164,9 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) iFiles, errStr, (long) maxFiles.rlim_max); ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); } +#ifdef USE_UNLIMITED_SELECT + glbl.SetFdSetSize(howmany(iFiles, __NFDBITS) * sizeof (fd_mask)); +#endif DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); finalize_it: @@ -2102,7 +2178,7 @@ static rsRetVal mainThread() * is still in its infancy (and not really done), we currently accept this issue. * rgerhards, 2009-06-29 */ - if(!(Debug || NoFork)) { + if(!(Debug == DEBUG_FULL || NoFork)) { close(1); close(2); bErrMsgToStderr = 0; @@ -2294,7 +2370,7 @@ doGlblProcessInit(void) thrdInit(); - if( !(Debug || NoFork) ) + if( !(Debug == DEBUG_FULL || NoFork) ) { DBGPRINTF("Checking pidfile.\n"); if (!check_pid(PidFile)) |
