diff options
276 files changed, 15175 insertions, 4280 deletions
@@ -1,19 +1,14 @@ -Rainer Gerhards <rgerhards@adiscon.com>, Adiscon GmbH -Michael Meckelein <mmeckelein@hq.adiscon.com>, Adiscon GmbH +Thankfully, we have had so many contributions that maintaining the +AUTHORS file would be a big task in itself. On the other hand, we +now use git and I make sure that each author receives proper credit +for patches I receive. -Contributors -Michael Biebl - - helped continously with autotools - - provided numerous advise on how to do things under Linux - - provided excellent advise in many, many cases -Andres Riancho (andres-dot-riancho-at-gmail-dot-com) - (alias APR in code files) - - supplied regexp functionality for the property replacer - a great feature. - thanks! -Bjoern Kalkbrenner - - provided code for the "execute shell script" action -Peter Vrabec - - provided IPv6-enabling code -varmojfekoj - - helped with a variety of things and, most importantly, contributed - the gssapi functionality +So rather than trying to reproduce the git author log here (and +often making mistakes in that), I invite you to check the git logs. +You can also do this online at + +http://git.adiscon.com/?p=rsyslog.git;a=summary + +Rainer Gerhards +<rgerhards@adiscon.com> +lead rsyslog developer @@ -1,4 +1,481 @@ --------------------------------------------------------------------------- +Version 5.5.7 [V5-BETA] (rgerhards), 2010-08-09 +- changed omudpspoof default spoof address to simplify typical use case + thanks to David Lang for suggesting this +- doc bugfix: pmlastmsg doc samples had errors +- bugfix[minor]: pmrfc3164sd had invalid name (resided in rsyslog name + space, what should not be the case for a contributed module) +- added omuxsock, which permits to write message to local Unix sockets + this is the counterpart to imuxsock, enabling fast local forwarding +--------------------------------------------------------------------------- +Version 5.5.6 [DEVEL] (rgerhards), 2010-07-21 +- added parser modules + * pmlastmsg, which supports the notoriously malformed "last message + repeated n times" messages from some syslogd's (namely sysklogd) + * pmrfc3164sd (contributed), supports RFC5424 structured data in + RFC3164 messages [untested] +- added new module type "string generator", used to speed up output + processing. Expected speedup for (typical) rsyslog processing is + roughly 5 to 6 percent compared to using string-based templates. + They may also be used to do more complex formatting with custom + C code, what provided greater flexibility and probably far higher + speed, for example if using multiple regular expressions within a + template. +- added 4 string generators for + * RSYSLOG_FileFormat + * RSYSLOG_TraditionalFileFormat + * RSYSLOG_ForwardFormat + * RSYSLOG_TraditionalForwardFormat +- bugfix: mutexes used to simulate atomic instructions were not destructed +- bugfix: regression caused more locking action in msg.c than necessary +- bugfix: "$ActionExecOnlyWhenPreviousIsSuspended on" was broken +- bugfix: segfault on HUP when "HUPIsRestart" was set to "on" + thanks varmojfekoj for the patch +- bugfix: default for $OMFileFlushOnTXEnd was wrong ("off"). + This, in default mode, caused buffered writing to be used, what + means that it looked like no output were written or partial + lines. Thanks to Michael Biebl for pointing out this bug. +- bugfix: programname filter in ! configuration can not be reset + Thanks to Kiss Gabor for the patch. +--------------------------------------------------------------------------- +Version 5.5.5 [DEVEL] (rgerhards), 2010-05-20 +- added new cancel-reduced action thread termination method + We now manage to cancel threads that block inside a retry loop to + terminate without the need to cancel the thread. Avoiding cancellation + helps keep the system complexity minimal and thus provides for better + stability. This also solves some issues with improper shutdown when + inside an action retry loop. +--------------------------------------------------------------------------- +Version 5.5.4 [DEVEL] (rgerhards), 2010-05-03 +- This version offers full support for Solaris on Intel and Sparc +- bugfix: problems with atomic operations emulation + replaced atomic operation emulation with new code. The previous code + seemed to have some issue and also limited concurrency severely. The + whole atomic operation emulation has been rewritten. +- bugfix: netstream ptcp support class was not correctly build on systems + without epoll() support +- bugfix: segfault on Solaris/Sparc +--------------------------------------------------------------------------- +Version 5.5.3 [DEVEL] (rgerhards), 2010-04-09 +- added basic but functional support for Solaris +- imported many bugfixes from 3.6.2/4.6.1 (see ChangeLog below!) +- added new property replacer option "date-rfc3164-buggyday" primarily + to ease migration from syslog-ng. See property replacer doc for + details. +- added capability to turn off standard LF delimiter in TCP server + via new directive "$InputTCPServerDisableLFDelimiter on" +- bugfix: failed to compile on systems without epoll support +- bugfix: comment char ('#') in literal terminated script parsing + and thus could not be used. + but tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=119 + [merged in from v3.22.2] +- imported patches from 4.6.0: + * improved testbench to contain samples for totally malformed messages + which miss parts of the message content + * bugfix: some malformed messages could lead to a missing LF inside files + or some other missing parts of the template content. + * bugfix: if a message ended immediately with a hostname, the hostname + was mistakenly interpreted as TAG, and localhost be used as hostname +--------------------------------------------------------------------------- +Version 5.5.2 [DEVEL] (rgerhards), 2010-02-05 +- applied patches that make rsyslog compile under Apple OS X. + Thanks to trey for providing these. +- replaced data type "bool" by "sbool" because this created some + portability issues. +- added $Escape8BitCharactersOnReceive directive + Thanks to David Lang for suggesting it. +- worked around an issue where omfile failed to compile on 32 bit platforms + under some circumstances (this smells like a gcc problem, but a simple + solution was available). Thanks to Kenneth Marshall for some advice. +- extended testbench +--------------------------------------------------------------------------- +Version 5.5.1 [DEVEL] (rgerhards), 2009-11-27 +- introduced the ablity for netstream drivers to utilize an epoll interface + This offers increased performance and removes the select() FDSET size + limit from imtcp. Note that we fall back to select() if there is no + epoll netstream drivers. So far, an epoll driver has only been + implemented for plain tcp syslog, the rest will follow once the code + proves well in practice AND there is demand. +- re-implemented $EscapeControlCharacterTab config directive + Based on Jonathan Bond-Caron's patch for v4. This now also includes some + automatted tests. +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +--------------------------------------------------------------------------- +Version 5.5.0 [DEVEL] (rgerhards), 2009-11-18 +- moved DNS resolution code out of imudp and into the backend processing + Most importantly, DNS resolution now never happens if the resolved name + is not required. Note that this applies to imudp - for the other inputs, + DNS resolution almost comes for free, so we do not do it there. However, + the new method has been implemented in a generic way and as such may + also be used by other modules in the future. +- added option to use unlimited-size select() calls + Thanks to varmjofekoj for the patch + This is not done in imudp, as it natively supports epoll(). +- doc: improved description of what loadable modules can do +--------------------------------------------------------------------------- +Version 5.4.2 [v5-stable] (rgerhards), 2010-03-?? +- bugfix(kind of): output plugin retry behaviour could cause engine to loop + The rsyslog engine did not guard itself against output modules that do + not properly convey back the tryResume() behaviour. This then leads to + what looks like an endless loop. I consider this to be a bug of the + engine not only because it should be hardened against plugin misbehaviour, + but also because plugins may not be totally able to avoid this situation + (depending on the type of and processing done by the plugin). +- bugfix: testbench failed when not executed in UTC+1 timezone + accidently, the time zone information was kept inside some + to-be-checked-for responses +- temporary bugfix replaced by permanent one for + message-induced off-by-one error (potential segfault) (see 4.6.2) + The analysis has been completed and a better fix been crafted and + integrated. +- bugfix(minor): status variable was uninitialized + However, this would have caused harm only if NO parser modules at + all were loaded, which would lead to a defunctional configuration + at all. And, even more important, this is impossible as two parser + modules are built-in and thus can not be "not loaded", so we always + have a minimum of two. +--------------------------------------------------------------------------- +Version 5.4.1 [v5-stable] (rgerhards), 2010-03-?? +- added new property replacer option "date-rfc3164-buggyday" primarily + to ease migration from syslog-ng. See property replacer doc for + details. [backport from 5.5.3 because urgently needed by some] +- imported all bugfixes vom 4.6.2 (see below) +--------------------------------------------------------------------------- +Version 5.4.0 [v5-stable] (rgerhards), 2010-03-08 +*************************************************************************** +* This is a new stable v5 version. It contains all fixes and enhancements * +* made during the 5.3.x phase as well as those listed below. * +* Note that the 5.2.x series was quite buggy and as such all users are * +* strongly advised to upgrade to 5.4.0. * +*************************************************************************** +- bugfix: omruleset failed to work in many cases + bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=179 + Thanks to Ryan B. Lynch for reporting this issue. +- bugfix: comment char ('#') in literal terminated script parsing + and thus could not be used. + but tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=119 + [merged in from v3.22.2] +--------------------------------------------------------------------------- +Version 5.3.7 [BETA] (rgerhards), 2010-01-27 +- bugfix: queues in direct mode could case a segfault, especially if an + action failed for action queues. The issue was an invalid increment of + a stack-based pointer which lead to destruction of the stack frame and + thus a segfault on function return. + Thanks to Michael Biebl for alerting us on this problem. +- bugfix: hostname accidently set to IP address for some message sources, + for example imudp. Thanks to Anton for reporting this bug. [imported v4] +- bugfix: ompgsql had problems with transaction support, what actually + rendered it unsuable. Thanks to forum user "horhe" for alerting me + on this bug and helping to debug/fix it! [imported from 5.3.6] +- bugfix: $CreateDirs variable not properly initialized, default thus + was random (but most often "on") [imported from v3] +- bugfix: potential segfaults during queue shutdown + (bugs require certain non-standard settings to appear) + Thanks to varmojfekoj for the patch [imported from 4.5.8] + [backport from 5.5.2] +- bugfix: wrong memory assignment for a config variable (probably + without causing any harm) [backport from 5.2.2] +- bugfix: rsyslog hangs when writing to a named pipe which nobody was + reading. Thanks to Michael Biebl for reporting this bug. + Bugzilla entry: http://bugzilla.adiscon.com/show_bug.cgi?id=169 + [imported from 4.5.8] +--------------------------------------------------------------------------- +Version 5.3.6 [BETA] (rgerhards), 2010-01-13 +- bugfix: ompgsql did not properly check the server connection in + tryResume(), which could lead to rsyslog running in a thight loop +- bugfix: suspension during beginTransaction() was not properly handled + by rsyslog core +- bugfix: omfile output was only written when buffer was full, not at + end of transaction +- bugfix: commit transaction was not properly conveyed to message layer, + potentially resulting in non-message destruction and thus hangs +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +- bugfix (kind of): check if TCP connection is still alive if using TLS + Thanks to Jonathan Bond-Caron for the patch. +- bugfix: $CreateDirs variable not properly initialized, default thus + was random (but most often "on") [imported from v3] +- bugfix: ompgsql had problems with transaction support, what actually + rendered it unsuable. Thanks to forum user "horhe" for alerting me + on this bug and helping to debug/fix it! +- bugfix: memory leak when sending messages in zip-compressed format + Thanks to Naoya Nakazawa for analyzing this issue and providing a patch. +- worked around an issue where omfile failed to compile on 32 bit platforms + under some circumstances (this smells like a gcc problem, but a simple + solution was available). Thanks to Kenneth Marshall for some advice. + [backported from 5.5.x branch] +--------------------------------------------------------------------------- +Version 5.3.5 [BETA] (rgerhards), 2009-11-13 +- some light performance enhancement by replacing time() call with much + faster (at least under linux) gettimeofday() calls. +- some improvement of omfile performance with dynafiles + saved costly time() calls by employing a logical clock, which is + sufficient for the use case +- bugfix: omudpspoof miscalculated source and destination ports + while this was probably not noticed for source ports, it resulted in + almost all destination ports being wrong, except for the default port + of 514, which by virtue of its binary representation was calculated + correct (and probably thus the bug not earlier detected). +- bugfixes imported from earlier releases + * bugfix: named pipes did no longer work (they always got an open error) + this was a regression from the omfile rewrite in 4.5.0 + * bugfix(testbench): sequence check was not always performed correctly, + that could result in tests reporting success when they actually failed +- improved testbench: added tests for UDP forwarding and omudpspoof +- doc bugfix: omudpspoof had wrong config command names ("om" missing) +- bugfix [imported from 4.4.3]: $ActionExecOnlyOnceEveryInterval did + not work. +- [inport v4] improved testbench, contains now tcp and gzip test cases +- [import v4] added a so-called "On Demand Debug" mode, in which debug + output can be generated only after the process has started, but not right + from the beginning. This is assumed to be useful for hard-to-find bugs. + Also improved the doc on the debug system. +- bugfix: segfault on startup when -q or -Q option was given + [imported from v3-stable] +--------------------------------------------------------------------------- +Version 5.3.4 [DEVEL] (rgerhards), 2009-11-04 +- added the ability to create custom message parsers +- added $RulesetParser config directive that permits to bind specific + parsers to specific rulesets +- added omruleset output module, which provides great flexibility in + action processing. THIS IS A VERY IMPORTANT ADDITION, see its doc + for why. +- added the capability to have ruleset-specific main message queues + This offers considerable additional flexibility AND superior performance + (in cases where multiple inputs now can avoid lock contention) +- bugfix: correct default for escape ('#') character restored + This was accidently changed to '\\', thanks to David Lang for reporting +- bugfix(testbench): testcase did not properly wait for rsyslogd shutdown + thus some unpredictable behavior and a false negative test result + could occur. +--------------------------------------------------------------------------- +Version 5.3.3 [DEVEL] (rgerhards), 2009-10-27 +- simplified and thus speeded up the queue engine, also fixed some + potential race conditions (in very unusual shutdown conditions) + along the way. The threading model has seriously changes, so there may + be some regressions. +- enhanced test environment (inlcuding testbench): support for enhancing + probability of memory addressing failure by using non-NULL default + value for malloced memory (optional, only if requested by configure + option). This helps to track down some otherwise undetected issues + within the testbench. +- bugfix: potential abort if inputname property was not set + primarily a problem of imdiag +- bugfix: message processing states were not set correctly in all cases + however, this had no negative effect, as the message processing state + was not evaluated when a batch was deleted, and that was the only case + where the state could be wrong. +--------------------------------------------------------------------------- +Version 5.3.2 [DEVEL] (rgerhards), 2009-10-21 +- enhanced omfile to support transactional interface. This will increase + performance in many cases. +- added multi-ruleset support to imudp +- re-enabled input thread termination handling that does avoid thread + cancellation where possible. This provides a more reliable mode of + rsyslogd termination (canceling threads my result in not properly + freed resouces and potential later hangs, even though we perform + proper cancel handling in our code). This is part of an effort to + reduce thread cancellation as much as possible in rsyslog. + NOTE: the code previously written code for this functionality had a + subtle race condition. The new code solves that. +- enhanced immark to support non-cancel input module termination +- improved imudp so that epoll can be used in more environments, + fixed potential compile time problem if EPOLL_CLOEXEC is not available. +- some cleanup/slight improvement: + * changed imuxsock to no longer use deprecated submitAndParseMsg() IF + * changed submitAndParseMsg() interface to be a wrapper around the new + way of message creation/submission. This enables older plugins to be + used together with the new interface. The removal also enables us to + drop a lot of duplicate code, reducing complexity and increasing + maintainability. +- bugfix: segfault when starting up with an invalid .qi file for a disk queue + Failed for both pure disk as well as DA queues. Now, we emit an error + message and disable disk queueing facility. +- bugfix: potential segfault on messages with empty MSG part. This was a + recently introduced regression. +- bugfix: debug string larger than 1K were improperly displayed. Max size + is now 32K, and if a string is even longer it is meaningfully truncated. +--------------------------------------------------------------------------- +Version 5.3.1 [DEVEL] (rgerhards), 2009-10-05 +- added $AbortOnUncleanConfig directive - permits to prevent startup when + there are problems with the configuration file. See it's doc for + details. +- included some important fixes from v4-stable: + * bugfix: invalid handling of zero-sized messages + * bugfix: zero-sized UDP messages are no longer processed + * bugfix: random data could be appended to message + * bugfix: reverse lookup reduction logic in imudp do DNS queries too often +- bugfixes imported from 4.5.4: + * bugfix: potential segfault in stream writer on destruction + * bugfix: potential race in object loader (obj.c) during use/release + * bugfixes: potential problems in out file zip writer +--------------------------------------------------------------------------- +Version 5.3.0 [DEVEL] (rgerhards), 2009-09-14 +- begun to add simple GUI programs to gain insight into running rsyslogd + instances and help setup and troubleshooting (active via the + --enable-gui ./configure switch) +- changed imudp to utilize epoll(), where available. This shall provide + slightly better performance (just slightly because we called select() + rather infrequently on a busy system) +--------------------------------------------------------------------------- +Version 5.2.2 [v5-stable] (rgerhards), 2009-11-?? +- bugfix: enabling GSSServer crashes rsyslog startup + Thanks to Tomas Kubina for the patch [imgssapi] +--------------------------------------------------------------------------- +Version 5.2.1 [v5-stable] (rgerhards), 2009-11-02 +- bugfix [imported from 4.4.3]: $ActionExecOnlyOnceEveryInterval did + not work. +- bugfix: segfault on startup when -q or -Q option was given + [imported from v3-stable] +--------------------------------------------------------------------------- +Version 5.2.0 [v5-stable] (rgerhards), 2009-11-02 +This is a re-release of version 5.1.6 as stable after we did not get any bug +reports during the whole beta phase. Still, this first v5-stable may not be +as stable as one hopes for, I am not sure if we did not get bug reports +just because nobody tried it. Anyhow, we need to go forward and so we +have the initial v5-stable. +--------------------------------------------------------------------------- +Version 5.1.6 [v5-beta] (rgerhards), 2009-10-15 +- feature imports from v4.5.6 +- bugfix: potential race condition when queue worker threads were + terminated +- bugfix: solved potential (temporary) stall of messages when the queue was + almost empty and few new data added (caused testbench to sometimes hang!) +- fixed some race condition in testbench +- added more elaborate diagnostics to parts of the testbench +- bugfixes imported from 4.5.4: + * bugfix: potential segfault in stream writer on destruction + * bugfix: potential race in object loader (obj.c) during use/release + * bugfixes: potential problems in out file zip writer +- included some important fixes from 4.4.2: + * bugfix: invalid handling of zero-sized messages + * bugfix: zero-sized UDP messages are no longer processed + * bugfix: random data could be appended to message + * bugfix: reverse lookup reduction logic in imudp do DNS queries too often +--------------------------------------------------------------------------- +Version 5.1.5 [v5-beta] (rgerhards), 2009-09-11 +- added new config option $ActionWriteAllMarkMessages + this option permites to process mark messages under all circumstances, + even if an action was recently called. This can be useful to use mark + messages as a kind of heartbeat. +- added new config option $InputUnixListenSocketCreatePath + to permit the auto-creation of pathes to additional log sockets. This + turns out to be useful if they reside on temporary file systems and + rsyslogd starts up before the daemons that create these sockets + (rsyslogd always creates the socket itself if it does not exist). +- added $LogRSyslogStatusMessages configuration directive + permitting to turn off rsyslog start/stop/HUP messages. See Debian + ticket http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=463793 +- bugfix: hostnames with dashes in them were incorrectly treated as + malformed, thus causing them to be treated as TAG (this was a regression + introduced from the "rfc3164 strict" change in 4.5.0). Testbench has been + updated to include a smaple message with a hostname containing a dash. +- bugfix: strings improperly reused, resulting in some message properties + be populated with strings from previous messages. This was caused by + an improper predicate check. +- added new config directive $omfileForceChown [import from 4.7.0] +--------------------------------------------------------------------------- +Version 5.1.4 [DEVEL] (rgerhards), 2009-08-20 +- legacy syslog parser changed so that it now accepts date stamps in + wrong case. Some devices seem to create them and I do not see any harm + in supporting that. +- added $InputTCPMaxListeners directive - permits to specify how many + TCP servers shall be possible (default is 20). +- bugfix: memory leak with some input modules. Those inputs that + use parseAndSubmitMsg() leak two small memory blocks with every message. + Typically, those process only relatively few messages, so the issue + does most probably not have any effect in practice. +- bugfix: if tcp listen port could not be created, no error message was + emitted +- bugfix: discard action did not work (did not discard messages) +- bugfix: discard action caused segfault +- bugfix: potential segfault in output file writer (omfile) + In async write mode, we use modular arithmetic to index the output + buffer array. However, the counter variables accidently were signed, + thus resulting in negative indizes after integer overflow. That in turn + could lead to segfaults, but was depending on the memory layout of + the instance in question (which in turn depended on a number of + variables, like compile settings but also configuration). The counters + are now unsigned (as they always should have been) and so the dangling + mis-indexing does no longer happen. This bug potentially affected all + installations, even if only some may actually have seen a segfault. +--------------------------------------------------------------------------- +Version 5.1.3 [DEVEL] (rgerhards), 2009-07-28 +- architecture change: queue now always has at least one worker thread + if not running in direct mode. Previous versions could run without + any active workers. This simplifies the code at a very small expense. + See v5 compatibility note document for more in-depth discussion. +- enhance: UDP spoofing supported via new output module omudpspoof + See the omudpspoof documentation for details and samples +- bugfix: message could be truncated after TAG, often when forwarding + This was a result of an internal processing error if maximum field + sizes had been specified in the property replacer. +- bugfix: minor static memory leak while reading configuration + did NOT leak based on message volume +- internal: added ability to terminate input modules not via pthread_cancel + but an alternate approach via pthread_kill. This is somewhat safer as we + do not need to think about the cancel-safeness of all libraries we use. + However, not all inputs can easily supported, so this now is a feature + that can be requested by the input module (the most important ones + request it). +--------------------------------------------------------------------------- +Version 5.1.2 [DEVEL] (rgerhards), 2009-07-08 +- bugfix: properties inputname, fromhost, fromhost-ip, msg were lost when + working with disk queues +- some performance enhancements +- bugfix: abort condition when RecvFrom was not set and message reduction + was on. Happend e.g. with imuxsock. +- added $klogConsoleLogLevel directive which permits to set a new + console log level while rsyslog is active +- some internal code cleanup +--------------------------------------------------------------------------- +Version 5.1.1 [DEVEL] (rgerhards), 2009-07-03 +- bugfix: huge memory leak in queue engine (made rsyslogd unusable in + production). Occured if at least one queue was in direct mode + (the default for action queues) +- imported many performance optimizations from v4-devel (4.5.0) +- bugfix: subtle (and usually irrelevant) issue in timout processing + timeout could be one second too early if nanoseconds wrapped +- set a more sensible timeout for shutdow, now 1.5 seconds to complete + processing (this also removes those cases where the shutdown message + was not written because the termination happened before it) +--------------------------------------------------------------------------- +Version 5.1.0 [DEVEL] (rgerhards), 2009-05-29 + +*********************************** NOTE ********************************** +The v5 versions of rsyslog feature a greatly redesigned queue engine. The +major theme for the v5 release is twofold: + +a) greatly improved performance +b) enable audit-grade processing + +Here, audit-grade processing means that rsyslog, if used together with +audit-grade transports and configured correctly, will never lose messages +that already have been acknowledged, not even in fatal failure cases like +sudden loss of power. + +Note that large parts of rsyslog's important core components have been +restructured to support these design goals. As such, early versions of +the engine will probably be less stable than the v3/v4 engine. + +Also note that the initial versions do not cover all and everything. As +usual, the code will evolve toward the final goal as version numbers +increase. +*********************************** NOTE ********************************** + +- redesigned queue engine so that it supports ultra-reliable operations + This resulted in a rewrite of large parts. The new capability can be + used to build audit-grade systems on the basis of rsyslog. +- added $MainMsgQueueDequeueBatchSize and $ActionQueueDequeueBatchSize + configuration directives +- implemented a new transactional output module interface which provides + superior performance (for databases potentially far superior performance) +- increased ompgsql performance by adapting to new transactional + output module interface +--------------------------------------------------------------------------- Version 4.7.3 [v4-devel] (rgerhards), 2010-??-?? - added omuxsock, which permits to write message to local Unix sockets this is the counterpart to imuxsock, enabling fast local forwarding @@ -47,9 +524,7 @@ Version 4.7.0 [v4-devel] (rgerhards), 2010-04-14 - imported changes from 4.5.7 and below - bugfix: potential segfault when -p command line option was used Thanks for varmojfekoj for pointing me at this bug. -- bugfix: potential segfaults during queue shutdown - (bugs require certain non-standard settings to appear) - Thanks to varmojfekoj for the patch [imported from 4.5.8] +- imported changes from 4.5.6 and below --------------------------------------------------------------------------- Version 4.6.4 [v4-stable] (rgerhards), 2010-08-05 - bugfix: zero-sized (empty) messages were processed by imtcp @@ -236,8 +711,7 @@ Version 4.5.8 [v4-beta] (rgerhards), 2010-02-10 Thanks to Jack for reporting this issue. - bugfix: rsyslog hang when writing to a named pipe which nobody was reading. Thanks to Michael Biebl for reporting this bug. -- bugfix: memory leak when sending messages in zip-compressed format - Thanks to Naoya Nakazawa for analyzing this issue and providing a patch. + Bugzilla entry: http://bugzilla.adiscon.com/show_bug.cgi?id=169 - bugfix: potential segfaults during queue shutdown (bugs require certain non-standard settings to appear) Thanks to varmojfekoj for the patch @@ -4676,6 +5150,21 @@ sysklogd maintainers for all their good work! --------------------------------------------------------------------------- ---------------------------------------------------------------------- +The following comments were left in the syslogd source. While they provide +not too much detail, the help to date when Rainer started work on the +project (which was 2003, now even surprising for Rainer himself ;)). + * \author Rainer Gerhards <rgerhards@adiscon.com> + * \date 2003-10-17 + * Some initial modifications on the sysklogd package to support + * liblogging. These have actually not yet been merged to the + * source you see currently (but they hopefully will) + * + * \date 2004-10-28 + * Restarted the modifications of sysklogd. This time, we + * focus on a simpler approach first. The initial goal is to + * provide MySQL database support (so that syslogd can log + * to the database). +---------------------------------------------------------------------- The following comments are from the stock syslogd.c source. They provide some insight into what happened to the source before we forked rsyslogd. However, much of the code already has been replaced and more diff --git a/Makefile.am b/Makefile.am index 111756f6..76a38322 100644 --- a/Makefile.am +++ b/Makefile.am @@ -99,6 +99,26 @@ if ENABLE_OMSTDOUT SUBDIRS += plugins/omstdout endif +if ENABLE_PMLASTMSG +SUBDIRS += plugins/pmlastmsg +endif + +if ENABLE_PMRFC3164SD +SUBDIRS += plugins/pmrfc3164sd +endif + +if ENABLE_OMRULESET +SUBDIRS += plugins/omruleset +endif + +if ENABLE_OMDBALERTING +SUBDIRS += plugins/omdbalerting +endif + +if ENABLE_OMUDPSPOOF +SUBDIRS += plugins/omudpspoof +endif + if ENABLE_OMUXSOCK SUBDIRS += plugins/omuxsock endif @@ -135,6 +155,10 @@ if ENABLE_ORACLE SUBDIRS += plugins/omoracle endif +if ENABLE_GUI +SUBDIRS += java +endif + # tests are added as last element, because tests may need different # modules that need to be generated first SUBDIRS += tests @@ -146,7 +170,26 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave --enable-extended-tests \ - --enable-omuxsock \ - --enable-imptcp +DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ + --enable-imfile \ + --enable-snmp \ + --enable-pgsql \ + --enable-libdbi \ + --enable-mysql \ + --enable-relp \ + --enable-rsyslogd \ + --enable-mail \ + --enable-klog \ + --enable-diagtools \ + --enable-gnutls \ + --enable-omstdout \ + --enable-pmlastmsg \ + --enable-omruleset \ + --enable-omprog \ + --enable-imdiag \ + --enable-imptcp \ + --enable-omuxsock \ + --enable-shave \ + --enable-extended-tests \ + --enable-memcheck ACLOCAL_AMFLAGS = -I m4 @@ -42,11 +42,19 @@ #include "cfsysline.h" #include "srUtils.h" #include "errmsg.h" +#include "batch.h" +#include "wti.h" #include "datetime.h" #include "unicode-helper.h" +#include "atomic.h" + +#define NO_TIME_PROVIDED 0 /* indicate we do not provide any cached time */ /* forward definitions */ -rsRetVal actionCallDoAction(action_t *pAction, msg_t *pMsg); +static rsRetVal processBatchMain(action_t *pAction, batch_t *pBatch, int*); +static rsRetVal doSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch); +static rsRetVal doSubmitToActionQNotAllMarkBatch(action_t *pAction, batch_t *pBatch); +static rsRetVal doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch); /* object static data (once for all instances) */ /* TODO: make this an object! DEFobjStaticHelpers -- rgerhards, 2008-03-05 */ @@ -55,16 +63,19 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +static int iActExecOnceInterval = 0; /* execute action once every nn seconds */ static int iActExecEveryNthOccur = 0; /* execute action every n-th occurence (0,1=always) */ static time_t iActExecEveryNthOccurTO = 0; /* timeout for n-occurence setting (in seconds, 0=never) */ static int glbliActionResumeInterval = 30; int glbliActionResumeRetryCount = 0; /* how often should suspended actions be retried? */ static int bActionRepMsgHasMsg = 0; /* last messsage repeated... has msg fragment in it */ +static int bActionWriteAllMarkMsgs = FALSE; /* should all mark messages be unconditionally written? */ static uchar *pszActionName; /* short name for the action */ -/* main message queue and its configuration parameters */ +/* action queue and its configuration parameters */ static queueType_t ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ static int iActionQueueSize = 1000; /* size of the main message queue above */ +static int iActionQueueDeqBatchSize = 16; /* batch size for action queues */ static int iActionQHighWtrMark = 800; /* high water mark for disk-assisted queues */ static int iActionQLowWtrMark = 200; /* low water mark for disk-assisted queues */ static int iActionQDiscardMark = 9800; /* begin to discard messages */ @@ -122,7 +133,7 @@ getActNow(action_t *pThis) { assert(pThis != NULL); if(pThis->tActNow == -1) { - pThis->tActNow = time(NULL); /* good time call - the only one done */ + pThis->tActNow = datetime.GetTime(NULL); /* good time call - the only one done */ if(pThis->tLastExec > pThis->tActNow) { /* if we are traveling back in time, reset tLastExec */ pThis->tLastExec = (time_t) 0; @@ -146,6 +157,7 @@ actionResetQueueParams(void) ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ iActionQueueSize = 1000; /* size of the main message queue above */ + iActionQueueDeqBatchSize = 16; /* default batch size */ iActionQHighWtrMark = 800; /* high water mark for disk-assisted queues */ iActionQLowWtrMark = 200; /* low water mark for disk-assisted queues */ iActionQDiscardMark = 9800; /* begin to discard messages */ @@ -179,7 +191,6 @@ actionResetQueueParams(void) */ rsRetVal actionDestruct(action_t *pThis) { - int i; DEFiRet; ASSERT(pThis != NULL); @@ -198,32 +209,6 @@ rsRetVal actionDestruct(action_t *pThis) d_free(pThis->pszName); d_free(pThis->ppTpl); - /* message ptr cleanup */ - for(i = 0 ; i < pThis->iNumTpls ; ++i) { - if(((uchar**)pThis->ppMsgs)[i] != NULL) { - switch(pThis->eParamPassing) { - case ACT_ARRAY_PASSING: -#if 0 /* later! */ - iArr = 0; - while(((char **)pThis->ppMsgs[i])[iArr] != NULL) { - d_free(((char **)pThis->ppMsgs[i])[iArr++]); - ((char **)pThis->ppMsgs[i])[iArr++] = NULL; - } - d_free(pThis->ppMsgs[i]); - pThis->ppMsgs[i] = NULL; -#endif - break; - case ACT_STRING_PASSING: - d_free(((uchar**)pThis->ppMsgs)[i]); - break; - default: - assert(0); - } - } - } - d_free(pThis->ppMsgs); - d_free(pThis->lenMsgs); - d_free(pThis); RETiRet; @@ -243,8 +228,9 @@ rsRetVal actionConstruct(action_t **ppThis) CHKmalloc(pThis = (action_t*) calloc(1, sizeof(action_t))); pThis->iResumeInterval = glbliActionResumeInterval; pThis->iResumeRetryCount = glbliActionResumeRetryCount; - pThis->tLastOccur = time(NULL); /* done once per action on startup only */ + pThis->tLastOccur = datetime.GetTime(NULL); /* done once per action on startup only */ pthread_mutex_init(&pThis->mutActExec, NULL); + INIT_ATOMIC_HELPER_MUT(pThis->mutCAS); SYNC_OBJ_TOOL_INIT(pThis); /* indicate we have a new action */ @@ -269,6 +255,32 @@ actionConstructFinalize(action_t *pThis) /* find a name for our queue */ snprintf((char*) pszQName, sizeof(pszQName)/sizeof(uchar), "action %d queue", iActionNbr); + /* now check if we can run the action in "firehose mode" during stage one of + * its processing (that is before messages are enqueued into the action q). + * This is only possible if some features, which require strict sequence, are + * not used. Thankfully, that is usually the case. The benefit of firehose + * mode is much faster processing (and simpler code) -- rgerhards, 2010-06-08 + */ + if( pThis->iExecEveryNthOccur > 1 + || pThis->f_ReduceRepeated + || pThis->iSecsExecOnceInterval + ) { + DBGPRINTF("info: firehose mode disabled for action because " + "iExecEveryNthOccur=%d, " + "ReduceRepeated=%d, " + "iSecsExecOnceInterval=%d\n", + pThis->iExecEveryNthOccur, pThis->f_ReduceRepeated, + pThis->iSecsExecOnceInterval + ); + pThis->submitToActQ = doSubmitToActionQComplexBatch; + } else if(pThis->bWriteAllMarkMsgs == FALSE) { + /* nearly full-speed submission mode, default case */ + pThis->submitToActQ = doSubmitToActionQNotAllMarkBatch; + } else { + /* full firehose submission mode */ + pThis->submitToActQ = doSubmitToActionQBatch; + } + /* we need to make a safety check: if the queue is NOT in direct mode, a single * message object may be accessed by multiple threads. As such, we need to enable * msg object thread safety in this case (this costs a bit performance and thus @@ -283,7 +295,8 @@ actionConstructFinalize(action_t *pThis) * to be run on multiple threads. So far, this is forbidden by the interface * spec. -- rgerhards, 2008-01-30 */ - CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, (rsRetVal (*)(void*,void*))actionCallDoAction)); + CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, + (rsRetVal (*)(void*, batch_t*, int*))processBatchMain)); obj.SetName((obj_t*) pThis->pQueue, pszQName); /* ... set some properties ... */ @@ -298,6 +311,7 @@ actionConstructFinalize(action_t *pThis) qqueueSetpUsr(pThis->pQueue, pThis); setQPROP(qqueueSetsizeOnDiskMax, "$ActionQueueMaxDiskSpace", iActionQueMaxDiskSpace); + setQPROP(qqueueSetiDeqBatchSize, "$ActionQueueDequeueBatchSize", iActionQueueDeqBatchSize); setQPROP(qqueueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); setQPROPstr(qqueueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); setQPROP(qqueueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); @@ -334,87 +348,271 @@ finalize_it: } -/* set an action back to active state -- rgerhards, 2007-08-02 + +/* set the global resume interval */ -static rsRetVal actionResume(action_t *pThis) +rsRetVal actionSetGlobalResumeInterval(int iNewVal) +{ + glbliActionResumeInterval = iNewVal; + return RS_RET_OK; +} + + +/* returns the action state name in human-readable form + * returned string must not be modified. + * rgerhards, 2009-05-07 + */ +static uchar *getActStateName(action_t *pThis) +{ + switch(pThis->eState) { + case ACT_STATE_RDY: + return (uchar*) "rdy"; + case ACT_STATE_ITX: + return (uchar*) "itx"; + case ACT_STATE_RTRY: + return (uchar*) "rtry"; + case ACT_STATE_SUSP: + return (uchar*) "susp"; + case ACT_STATE_DIED: + return (uchar*) "died"; + case ACT_STATE_COMM: + return (uchar*) "comm"; + default: + return (uchar*) "ERROR/UNKNWON"; + } +} + + +/* returns a suitable return code based on action state + * rgerhards, 2009-05-07 + */ +static rsRetVal getReturnCode(action_t *pThis) { DEFiRet; ASSERT(pThis != NULL); - pThis->bSuspended = 0; + switch(pThis->eState) { + case ACT_STATE_RDY: + iRet = RS_RET_OK; + break; + case ACT_STATE_ITX: + if(pThis->bHadAutoCommit) { + pThis->bHadAutoCommit = 0; /* auto-reset */ + iRet = RS_RET_PREVIOUS_COMMITTED; + } else { + iRet = RS_RET_DEFER_COMMIT; + } + break; + case ACT_STATE_RTRY: + iRet = RS_RET_SUSPENDED; + break; + case ACT_STATE_SUSP: + case ACT_STATE_DIED: + iRet = RS_RET_ACTION_FAILED; + break; + default: + DBGPRINTF("Invalid action engine state %d, program error\n", + (int) pThis->eState); + iRet = RS_RET_ERR; + break; + } RETiRet; } -/* set the global resume interval +/* set the action to a new state + * rgerhards, 2007-08-02 */ -rsRetVal actionSetGlobalResumeInterval(int iNewVal) +static inline void actionSetState(action_t *pThis, action_state_t newState) { - glbliActionResumeInterval = iNewVal; - return RS_RET_OK; + pThis->eState = newState; + DBGPRINTF("Action %p transitioned to state: %s\n", pThis, getActStateName(pThis)); +} + +/* Handles the transient commit state. So far, this is + * mostly a dummy... + * rgerhards, 2007-08-02 + */ +static void actionCommitted(action_t *pThis) +{ + actionSetState(pThis, ACT_STATE_RDY); +} + + +/* set action to "rtry" state. + * rgerhards, 2007-08-02 + */ +static void actionRetry(action_t *pThis) +{ + actionSetState(pThis, ACT_STATE_RTRY); + pThis->iResumeOKinRow++; +} + + +/* Disable action, this means it will never again be usable + * until rsyslog is reloaded. Use only as a last resort, but + * depends on output module. + * rgerhards, 2007-08-02 + */ +static void actionDisable(action_t *pThis) +{ + actionSetState(pThis, ACT_STATE_DIED); +} + + +/* Suspend action, this involves changing the acton state as well + * as setting the next retry time. + * if we have more than 10 retries, we prolong the + * retry interval. If something is really stalled, it will + * get re-tried only very, very seldom - but that saves + * CPU time. TODO: maybe a config option for that? + * rgerhards, 2007-08-02 + */ +static inline void actionSuspend(action_t *pThis, time_t ttNow) +{ + if(ttNow == NO_TIME_PROVIDED) + datetime.GetTime(&ttNow); + pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); + actionSetState(pThis, ACT_STATE_SUSP); + DBGPRINTF("earliest retry=%d\n", (int) pThis->ttResumeRtry); } -/* suspend an action -- rgerhards, 2007-08-02 +/* actually do retry processing. Note that the function receives a timestamp so + * that we do not need to call the (expensive) time() API. + * Note that we do the full retry processing here, doing the configured number of + * iterations. -- rgerhards, 2009-05-07 + * We need to guard against module which always return RS_RET_OK from their tryResume() + * entry point. This is invalid, but has harsh consequences: it will cause the rsyslog + * engine to go into a tight loop. That obviously is not acceptable. As such, we track the + * count of iterations that a tryResume returning RS_RET_OK is immediately followed by + * an unsuccessful call to doAction(). If that happens more than 1,000 times, we assume + * the return acutally is a RS_RET_SUSPENDED. In order to go through the various + * resumption stages, we do this for every 1000 requests. This magic number 1000 may + * not be the most appropriate, but it should be thought of a "if nothing else helps" + * kind of facility: in the first place, the module should return a proper indication + * of its inability to recover. -- rgerhards, 2010-04-26. */ -static rsRetVal actionSuspend(action_t *pThis, time_t tNow) +static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) { + int iRetries; + int iSleepPeriod; + int bTreatOKasSusp; DEFiRet; ASSERT(pThis != NULL); - pThis->bSuspended = 1; - pThis->ttResumeRtry = tNow + pThis->iResumeInterval; - pThis->iNbrResRtry = 0; /* tell that we did not yet retry to resume */ + iRetries = 0; + while((*pThis->pbShutdownImmediate == 0) && pThis->eState == ACT_STATE_RTRY) { + iRet = pThis->pMod->tryResume(pThis->pModData); + if((pThis->iResumeOKinRow > 999) && (pThis->iResumeOKinRow % 1000 == 0)) { + bTreatOKasSusp = 1; + pThis->iResumeOKinRow = 0; + } else { + bTreatOKasSusp = 0; + } + if((iRet == RS_RET_OK) && (!bTreatOKasSusp)) { + actionSetState(pThis, ACT_STATE_RDY); + } else if(iRet == RS_RET_SUSPENDED || bTreatOKasSusp) { + /* max retries reached? */ + if((pThis->iResumeRetryCount != -1 && iRetries >= pThis->iResumeRetryCount)) { + actionSuspend(pThis, ttNow); + } else { + ++pThis->iNbrResRtry; + ++iRetries; + iSleepPeriod = pThis->iResumeInterval; + ttNow += iSleepPeriod; /* not truly exact, but sufficiently... */ + srSleep(iSleepPeriod, 0); + if(*pThis->pbShutdownImmediate) { + ABORT_FINALIZE(RS_RET_FORCE_TERM); + } + } + } else if(iRet == RS_RET_DISABLE_ACTION) { + actionDisable(pThis); + } + } + + if(pThis->eState == ACT_STATE_RDY) { + pThis->iNbrResRtry = 0; + } + +finalize_it: RETiRet; } /* try to resume an action -- rgerhards, 2007-08-02 - * returns RS_RET_OK if resumption worked, RS_RET_SUSPEND if the - * action is still suspended. + * changed to new action state engine -- rgerhards, 2009-05-07 */ static rsRetVal actionTryResume(action_t *pThis) { DEFiRet; - time_t ttNow; + time_t ttNow = NO_TIME_PROVIDED; ASSERT(pThis != NULL); - /* for resume handling, we must always obtain a fresh timestamp. We used - * to use the action timestamp, but in this case we will never reach a - * point where a resumption is actually tried, because the action timestamp - * is always in the past. So we can not avoid doing a fresh time() call - * here. -- rgerhards, 2009-03-18 - */ - time(&ttNow); /* cache "now" */ - - /* first check if it is time for a re-try */ - if(ttNow > pThis->ttResumeRtry) { - iRet = pThis->pMod->tryResume(pThis->pModData); - if(iRet == RS_RET_SUSPENDED) { - /* set new tryResume time */ - ++pThis->iNbrResRtry; - /* if we have more than 10 retries, we prolong the - * retry interval. If something is really stalled, it will - * get re-tried only very, very seldom - but that saves - * CPU time. TODO: maybe a config option for that? - * rgerhards, 2007-08-02 - */ - pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); + if(pThis->eState == ACT_STATE_SUSP) { + /* if we are suspended, we need to check if the timeout expired. + * for this handling, we must always obtain a fresh timestamp. We used + * to use the action timestamp, but in this case we will never reach a + * point where a resumption is actually tried, because the action timestamp + * is always in the past. So we can not avoid doing a fresh time() call + * here. -- rgerhards, 2009-03-18 + */ + datetime.GetTime(&ttNow); /* cache "now" */ + if(ttNow > pThis->ttResumeRtry) { + actionSetState(pThis, ACT_STATE_RTRY); /* back to retries */ } - } else { - /* it's too early, we are still suspended --> indicate this */ - iRet = RS_RET_SUSPENDED; } - if(iRet == RS_RET_OK) - actionResume(pThis); + if(pThis->eState == ACT_STATE_RTRY) { + if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */ + datetime.GetTime(&ttNow); + CHKiRet(actionDoRetry(pThis, ttNow)); + } + + if(Debug && (pThis->eState == ACT_STATE_RTRY ||pThis->eState == ACT_STATE_SUSP)) { + DBGPRINTF("actionTryResume: action state: %s, next retry (if applicable): %u [now %u]\n", + getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + } + +finalize_it: + RETiRet; +} + + +/* prepare an action for performing work. This involves trying to recover it, + * depending on its current state. + * rgerhards, 2009-05-07 + */ +static inline rsRetVal actionPrepare(action_t *pThis) +{ + DEFiRet; - DBGPRINTF("actionTryResume: iRet: %d, next retry (if applicable): %u [now %u]\n", - iRet, (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + assert(pThis != NULL); + CHKiRet(actionTryResume(pThis)); + /* if we are now ready, we initialize the transaction and advance + * action state accordingly + */ + if(pThis->eState == ACT_STATE_RDY) { + iRet = pThis->pMod->mod.om.beginTransaction(pThis->pModData); + switch(iRet) { + case RS_RET_OK: + actionSetState(pThis, ACT_STATE_ITX); + break; + case RS_RET_SUSPENDED: + actionRetry(pThis); + break; + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + default:FINALIZE; + } + } + +finalize_it: RETiRet; } @@ -425,129 +623,461 @@ static rsRetVal actionTryResume(action_t *pThis) rsRetVal actionDbgPrint(action_t *pThis) { DEFiRet; + char *sz; dbgprintf("%s: ", module.GetStateName(pThis->pMod)); pThis->pMod->dbgPrintInstInfo(pThis->pModData); dbgprintf("\n\tInstance data: 0x%lx\n", (unsigned long) pThis->pModData); dbgprintf("\tRepeatedMsgReduction: %d\n", pThis->f_ReduceRepeated); dbgprintf("\tResume Interval: %d\n", pThis->iResumeInterval); - dbgprintf("\tSuspended: %d", pThis->bSuspended); - if(pThis->bSuspended) { - dbgprintf(" next retry: %u, number retries: %d", (unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry); + if(pThis->eState == ACT_STATE_SUSP) { + dbgprintf("\tresume next retry: %u, number retries: %d", + (unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry); } - dbgprintf("\n"); - dbgprintf("\tDisabled: %d\n", !pThis->bEnabled); + dbgprintf("\tState: %s\n", getActStateName(pThis)); dbgprintf("\tExec only when previous is suspended: %d\n", pThis->bExecWhenPrevSusp); + if(pThis->submitToActQ == doSubmitToActionQComplexBatch) { + sz = "slow, but feature-rich"; + } else if(pThis->submitToActQ == doSubmitToActionQNotAllMarkBatch) { + sz = "fast, but supports partial mark messages"; + } else if(pThis->submitToActQ == doSubmitToActionQBatch) { + sz = "firehose (fastest)"; + } else { + sz = "unknown (need to update debug display?)"; + } + dbgprintf("\tsubmission mode: %s\n", sz); dbgprintf("\n"); RETiRet; } -/* call the DoAction output plugin entry point - * rgerhards, 2008-01-28 +/* prepare the calling parameters for doAction() + * rgerhards, 2009-05-07 */ -#pragma GCC diagnostic ignored "-Wempty-body" -rsRetVal -actionCallDoAction(action_t *pAction, msg_t *pMsg) +static rsRetVal prepareDoActionParams(action_t *pAction, msg_t *pMsg, uchar **ppMsgs, size_t *lenMsgs) { - DEFiRet; - int iRetries; int i; - int iArr; - int iSleepPeriod; - int bCallAction; - int iCancelStateSave; + DEFiRet; ASSERT(pAction != NULL); - /* We now must guard the output module against execution by multiple threads. The - * plugin interface specifies that output modules must not be thread-safe (except - * if they notify us they are - functionality not yet implemented...). - * rgerhards, 2008-01-30 - */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pAction->mutActExec); - pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - pthread_setcancelstate(iCancelStateSave, NULL); - /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { switch(pAction->eParamPassing) { case ACT_STRING_PASSING: - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(((uchar**)pAction->ppMsgs)[i]), &(pAction->lenMsgs[i]))); + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]), &lenMsgs[i])); break; case ACT_ARRAY_PASSING: - CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(((uchar**)pAction->ppMsgs)[i]))); + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(ppMsgs[i]))); + break; + case ACT_MSG_PASSING: + /* we abuse the uchar* ptr, it now actually is a void*, but we can not + * change that other than by chaning the interface, what we don't like... + */ + ppMsgs[i] = (void*) pMsg; + lenMsgs[i] = 0; /* init for *next* action */ break; default:assert(0); /* software bug if this happens! */ } } - iRetries = 0; - do { - /* on first invocation, this if should never be true. We just put it at the top - * of the loop so that processing (and code) is simplified. This code is actually - * triggered on the 2nd+ invocation. -- rgerhards, 2008-01-30 - */ - if(iRet == RS_RET_SUSPENDED) { - /* ok, this calls for our retry logic... */ - ++iRetries; - iSleepPeriod = pAction->iResumeInterval; - srSleep(iSleepPeriod, 0); - } - /* first check if we are suspended and, if so, retry */ - if(actionIsSuspended(pAction)) { - iRet = actionTryResume(pAction); - if(iRet == RS_RET_OK) - bCallAction = 1; - else - bCallAction = 0; - } else { - bCallAction = 1; - } +finalize_it: + RETiRet; +} + - if(bCallAction) { - /* call configured action */ - iRet = pAction->pMod->mod.om.doAction(pAction->ppMsgs, pMsg->msgFlags, pAction->pModData); - if(iRet == RS_RET_SUSPENDED) { - DBGPRINTF("Action requested to be suspended, done that.\n"); - actionSuspend(pAction, getActNow(pAction)); +/* cleanup doAction calling parameters + * rgerhards, 2009-05-07 + */ +static rsRetVal cleanupDoActionParams(action_t *pAction, uchar ***ppMsgs) +{ + int iArr; + int i; + DEFiRet; + + ASSERT(pAction != NULL); + + for(i = 0 ; i < pAction->iNumTpls ; ++i) { + if(((uchar**)ppMsgs)[i] != NULL) { + iArr = 0; + while((((uchar***)ppMsgs)[i][iArr]) != NULL) { + d_free(((uchar ***)ppMsgs)[i][iArr++]); + ((uchar ***)ppMsgs)[i][iArr++] = NULL; } + d_free(((uchar**)ppMsgs)[i]); + ((uchar**)ppMsgs)[i] = NULL; } + } + + RETiRet; +} - } while(iRet == RS_RET_SUSPENDED && (pAction->iResumeRetryCount == -1 || iRetries < pAction->iResumeRetryCount)); /* do...while! */ - if(iRet == RS_RET_DISABLE_ACTION) { - DBGPRINTF("Action requested to be disabled, done that.\n"); - pAction->bEnabled = 0; /* that's it... */ +/* call the DoAction output plugin entry point + * rgerhards, 2008-01-28 + */ +rsRetVal +actionCallDoAction(action_t *pThis, msg_t *pMsg, void *actParams) +{ + int i; + DEFiRet; + + ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pMsg, msg); + + DBGPRINTF("entering actionCalldoAction(), state: %s\n", getActStateName(pThis)); + + pThis->bHadAutoCommit = 0; +//d_pthread_mutex_lock(&pThis->mutActExec); +//pthread_cleanup_push(mutexCancelCleanup, &pThis->mutActExec); + iRet = pThis->pMod->mod.om.doAction(actParams, pMsg->msgFlags, pThis->pModData); +//pthread_cleanup_pop(1); /* unlock mutex */ + switch(iRet) { + case RS_RET_OK: + actionCommitted(pThis); + pThis->iResumeOKinRow = 0; /* we had a successful call! */ + break; + case RS_RET_DEFER_COMMIT: + pThis->iResumeOKinRow = 0; /* we had a successful call! */ + /* we are done, action state remains the same */ + break; + case RS_RET_PREVIOUS_COMMITTED: + /* action state remains the same, but we had a commit. */ + pThis->bHadAutoCommit = 1; + pThis->iResumeOKinRow = 0; /* we had a successful call! */ + break; + case RS_RET_SUSPENDED: + actionRetry(pThis); + break; + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + default:/* permanent failure of this message - no sense in retrying. This is + * not yet handled (but easy TODO) + */ + FINALIZE; } + iRet = getReturnCode(pThis); finalize_it: - /* cleanup */ - for(i = 0 ; i < pAction->iNumTpls ; ++i) { - if(((uchar**)pAction->ppMsgs)[i] != NULL) { - switch(pAction->eParamPassing) { - case ACT_ARRAY_PASSING: - iArr = 0; - while((((uchar***)pAction->ppMsgs)[i][iArr]) != NULL) { - d_free(((uchar ***)pAction->ppMsgs)[i][iArr++]); - ((uchar ***)pAction->ppMsgs)[i][iArr++] = NULL; + + /* we need to cleanup the batches string buffers if they have been used + * in a non-standard way. -- rgerhards, 2010-06-15 + * Note that we may do this at the batch level, this would provide a bit + * more concurrency (TODO). + */ + switch(pThis->eParamPassing) { + case ACT_STRING_PASSING: + /* nothing to do in that case */ + break; + case ACT_ARRAY_PASSING: + cleanupDoActionParams(pThis, actParams); /* iRet ignored! */ + break; + case ACT_MSG_PASSING: + /* nothing to do in that case */ + for(i = 0 ; i < pThis->iNumTpls ; ++i) { + ((uchar**)actParams)[i] = NULL; + } + break; + } + + RETiRet; +} + + +/* process a message + * this readies the action and then calls doAction() + * rgerhards, 2008-01-28 + */ +static inline rsRetVal +actionProcessMessage(action_t *pThis, msg_t *pMsg, void *actParams) +{ + DEFiRet; + + ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pMsg, msg); + + CHKiRet(actionPrepare(pThis)); + if(pThis->eState == ACT_STATE_ITX) + CHKiRet(actionCallDoAction(pThis, pMsg, actParams)); + + iRet = getReturnCode(pThis); +finalize_it: + RETiRet; +} + + +/* finish processing a batch. Most importantly, that means we commit if we + * need to do so. + * rgerhards, 2008-01-28 + */ +static rsRetVal +finishBatch(action_t *pThis, batch_t *pBatch) +{ + int i; + DEFiRet; + + ASSERT(pThis != NULL); + + if(pThis->eState == ACT_STATE_RDY) { + /* we just need to flag the batch as commited */ + FINALIZE; /* nothing to do */ + } + + CHKiRet(actionPrepare(pThis)); + if(pThis->eState == ACT_STATE_ITX) { + iRet = pThis->pMod->mod.om.endTransaction(pThis->pModData); + switch(iRet) { + case RS_RET_OK: + actionCommitted(pThis); + /* flag messages as committed */ + for(i = 0 ; i < pBatch->nElem ; ++i) { + batchSetElemState(pBatch, i, BATCH_STATE_COMM); + pBatch->pElem[i].bPrevWasSuspended = 0; /* we had success! */ } - d_free(((uchar**)pAction->ppMsgs)[i]); - ((uchar**)pAction->ppMsgs)[i] = NULL; break; - case ACT_STRING_PASSING: + case RS_RET_SUSPENDED: + actionRetry(pThis); break; - default: - assert(0); + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + case RS_RET_DEFER_COMMIT: + DBGPRINTF("output plugin error: endTransaction() returns RS_RET_DEFER_COMMIT " + "- ignored\n"); + actionCommitted(pThis); + break; + case RS_RET_PREVIOUS_COMMITTED: + DBGPRINTF("output plugin error: endTransaction() returns RS_RET_PREVIOUS_COMMITTED " + "- ignored\n"); + actionCommitted(pThis); + break; + default:/* permanent failure of this message - no sense in retrying. This is + * not yet handled (but easy TODO) + */ + FINALIZE; + } + } + iRet = getReturnCode(pThis); + +finalize_it: + RETiRet; +} + + +/* try to submit a partial batch of elements. + * rgerhards, 2009-05-12 + */ +static inline rsRetVal +tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem) +{ + int i; + int iElemProcessed; + int iCommittedUpTo; + msg_t *pMsg; + rsRetVal localRet; + DEFiRet; + + assert(pBatch != NULL); + assert(pnElem != NULL); + + i = pBatch->iDoneUpTo; /* all messages below that index are processed */ + iElemProcessed = 0; + iCommittedUpTo = i; + while(iElemProcessed <= *pnElem && i < pBatch->nElem) { + if(*(pBatch->pbShutdownImmediate)) + ABORT_FINALIZE(RS_RET_FORCE_TERM); + if( pBatch->pElem[i].bFilterOK + && pBatch->pElem[i].state != BATCH_STATE_DISC + && ((pAction->bExecWhenPrevSusp == 0) || pBatch->pElem[i].bPrevWasSuspended) ) { + pMsg = (msg_t*) pBatch->pElem[i].pUsrp; + localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams); + DBGPRINTF("action call returned %d\n", localRet); + /* Note: we directly modify the batch object state, because we know that + * wo do not overwrite BATCH_STATE_DISC indicators! + */ + if(localRet == RS_RET_OK) { + /* mark messages as committed */ + while(iCommittedUpTo <= i) { + pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; + } + } else if(localRet == RS_RET_PREVIOUS_COMMITTED) { + /* mark messages as committed */ + while(iCommittedUpTo < i) { + pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; + } + pBatch->pElem[i].state = BATCH_STATE_SUB; + } else if(localRet == RS_RET_DEFER_COMMIT) { + pBatch->pElem[i].state = BATCH_STATE_SUB; + } else if(localRet == RS_RET_DISCARDMSG) { + pBatch->pElem[i].state = BATCH_STATE_DISC; + } else { + dbgprintf("tryDoAction: unexpected error code %d[nElem %d, Commited UpTo %d], finalizing\n", + localRet, *pnElem, iCommittedUpTo); + iRet = localRet; + FINALIZE; } } + ++i; + ++iElemProcessed; } +finalize_it: + if(pBatch->iDoneUpTo != iCommittedUpTo) { + pBatch->iDoneUpTo = iCommittedUpTo; + } + RETiRet; +} + + +/* submit a batch for actual action processing. + * The first nElem elements are processed. This function calls itself + * recursively if it needs to handle errors. + * Note: we don't need the number of the first message to be processed as a parameter, + * because this is kept track of inside the batch itself (iDoneUpTo). + * rgerhards, 2009-05-12 + */ +static rsRetVal +submitBatch(action_t *pAction, batch_t *pBatch, int nElem) +{ + int i; + int bDone; + rsRetVal localRet; + DEFiRet; + + assert(pBatch != NULL); + + bDone = 0; + do { + localRet = tryDoAction(pAction, pBatch, &nElem); + if(localRet == RS_RET_FORCE_TERM) { + ABORT_FINALIZE(RS_RET_FORCE_TERM); + } + if( localRet == RS_RET_OK + || localRet == RS_RET_PREVIOUS_COMMITTED + || localRet == RS_RET_DEFER_COMMIT) { + /* try commit transaction, once done, we can simply do so as if + * that return state was returned from tryDoAction(). + */ + localRet = finishBatch(pAction, pBatch); + } + + if( localRet == RS_RET_OK + || localRet == RS_RET_PREVIOUS_COMMITTED + || localRet == RS_RET_DEFER_COMMIT) { + bDone = 1; + } else if(localRet == RS_RET_SUSPENDED) { + ; /* do nothing, this will retry the full batch */ + } else if(localRet == RS_RET_ACTION_FAILED) { + /* in this case, everything not yet committed is BAD */ + for(i = pBatch->iDoneUpTo ; i < nElem ; ++i) { + if( pBatch->pElem[i].state != BATCH_STATE_DISC + && pBatch->pElem[i].state != BATCH_STATE_COMM ) { + pBatch->pElem[i].state = BATCH_STATE_BAD; + pBatch->pElem[i].bPrevWasSuspended = 1; + } + } + bDone = 1; + } else { + if(nElem == 1) { + batchSetElemState(pBatch, i, BATCH_STATE_BAD); + bDone = 1; + } else { + /* retry with half as much. Depth is log_2 batchsize, so recursion is not too deep */ + submitBatch(pAction, pBatch, nElem / 2); + submitBatch(pAction, pBatch, nElem - (nElem / 2)); + bDone = 1; + } + } + } while(!bDone && !*(pBatch->pbShutdownImmediate)); /* do .. while()! */ + + if(*(pBatch->pbShutdownImmediate)) + ABORT_FINALIZE(RS_RET_FORCE_TERM); + +finalize_it: + RETiRet; +} + + + +/* The following function prepares a batch for processing, that it is + * reinitializes batch states, generates strings and does everything else + * that needs to be done in order to make the batch ready for submission to + * the actual output module. Note that we look at the precomputed + * filter OK condition and process only those messages, that actually matched + * the filter. + * rgerhards, 2010-06-14 + */ +static inline rsRetVal +prepareBatch(action_t *pAction, batch_t *pBatch) +{ + int i; + batch_obj_t *pElem; + DEFiRet; + + pBatch->iDoneUpTo = 0; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + pElem = &(pBatch->pElem[i]); + if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) { + pElem->state = BATCH_STATE_RDY; + prepareDoActionParams(pAction, (msg_t*) pElem->pUsrp, + (uchar**) &(pElem->staticActParams), pElem->staticLenParams); + } + } + RETiRet; +} + + +/* receive a batch and process it. This includes retry handling. + * rgerhards, 2009-05-12 + */ +static inline rsRetVal +processAction(action_t *pAction, batch_t *pBatch) +{ + DEFiRet; + + assert(pBatch != NULL); + CHKiRet(submitBatch(pAction, pBatch, pBatch->nElem)); + iRet = finishBatch(pAction, pBatch); + +finalize_it: + RETiRet; +} + + +#pragma GCC diagnostic ignored "-Wempty-body" +/* receive an array of to-process user pointers and submit them + * for processing. + * rgerhards, 2009-04-22 + */ +static rsRetVal +processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) +{ + int *pbShutdownImmdtSave; + DEFiRet; + + assert(pBatch != NULL); + + pbShutdownImmdtSave = pBatch->pbShutdownImmediate; + pBatch->pbShutdownImmediate = pbShutdownImmediate; + pAction->pbShutdownImmediate = pBatch->pbShutdownImmediate; + CHKiRet(prepareBatch(pAction, pBatch)); + + /* We now must guard the output module against execution by multiple threads. The + * plugin interface specifies that output modules must not be thread-safe (except + * if they notify us they are - functionality not yet implemented...). + * rgerhards, 2008-01-30 + */ + d_pthread_mutex_lock(&pAction->mutActExec); + pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); + + iRet = processAction(pAction, pBatch); + pthread_cleanup_pop(1); /* unlock mutex */ - msgDestruct(&pMsg); /* we are now finished with the message */ +finalize_it: + pBatch->pbShutdownImmediate = pbShutdownImmdtSave; RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" @@ -563,7 +1093,6 @@ rsRetVal actionCallHUPHdlr(action_t *pAction) { DEFiRet; - int iCancelStateSave; ASSERT(pAction != NULL); DBGPRINTF("Action %p checks HUP hdlr: %p\n", pAction, pAction->pMod->doHUP); @@ -572,10 +1101,8 @@ actionCallHUPHdlr(action_t *pAction) FINALIZE; /* no HUP handler, so we are done ;) */ } - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); d_pthread_mutex_lock(&pAction->mutActExec); pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - pthread_setcancelstate(iCancelStateSave, NULL); CHKiRet(pAction->pMod->doHUP(pAction->pModData)); pthread_cleanup_pop(1); /* unlock mutex */ @@ -615,20 +1142,8 @@ static rsRetVal setActionQueType(void __attribute__((unused)) *pVal, uchar *pszT } -/* rgerhards 2004-11-09: fprintlog() is the actual driver for - * the output channel. It receives the channel description (f) as - * well as the message and outputs them according to the channel - * semantics. The message is typically already contained in the - * channel save buffer (f->f_prevline). This is not only the case - * when a message was already repeated but also when a new message - * arrived. - * rgerhards 2007-08-01: interface changed to use action_t - * rgerhards, 2007-12-11: please note: THIS METHOD MUST ONLY BE - * CALLED AFTER THE CALLER HAS LOCKED THE pAction OBJECT! We do - * not do this here. Failing to do so results in all kinds of - * "interesting" problems! - * RGERHARDS, 2008-01-29: - * This is now the action caller and has been renamed. +/* This function builds up a batch of messages to be (later) + * submitted to the action queue. */ rsRetVal actionWriteToAction(action_t *pAction) @@ -754,28 +1269,16 @@ finalize_it: /* helper to actonCallAction, mostly needed because of this damn * pthread_cleanup_push() POSIX macro... */ -static rsRetVal +static inline rsRetVal doActionCallAction(action_t *pAction, msg_t *pMsg) { DEFiRet; - /* first, we need to check if this is a disabled - * entry. If so, we must not further process it. - * rgerhards 2005-09-26 - * In the future, disabled modules may be re-probed from time - * to time. They are in a perfectly legal state, except that the - * doAction method indicated that it wanted to be disabled - but - * we do not consider this is a solution for eternity... So we - * should check from time to time if affairs have improved. - * rgerhards, 2007-07-24 - */ - if(pAction->bEnabled == 0) { - ABORT_FINALIZE(RS_RET_OK); - } pAction->tActNow = -1; /* we do not yet know our current time (clear prev. value) */ /* don't output marks to recently written outputs */ - if((pMsg->msgFlags & MARK) && (getActNow(pAction) - pAction->f_time) < MarkInterval / 2) { + if(pAction->bWriteAllMarkMsgs == FALSE + && (pMsg->msgFlags & MARK) && (getActNow(pAction) - pAction->f_time) < MarkInterval / 2) { ABORT_FINALIZE(RS_RET_OK); } @@ -821,77 +1324,167 @@ finalize_it: RETiRet; } -/* call the configured action. Does all necessary housekeeping. - * rgerhards, 2007-08-01 - * FYI: currently, this function is only called from the queue - * consumer. So we (conceptually) run detached from the input - * threads (which also means we may run much later than when the - * message was generated). +/* This submits the message to the action queue in case we do NOT need to handle repeat + * message processing. That case permits us to gain lots of freedom during processing + * and thus speed. + * rgerhards, 2010-06-08 */ -#pragma GCC diagnostic ignored "-Wempty-body" -rsRetVal -actionCallAction(action_t *pAction, msg_t *pMsg) +static inline rsRetVal +doSubmitToActionQ(action_t *pAction, msg_t *pMsg) { DEFiRet; - int iCancelStateSave; - ISOBJ_TYPE_assert(pMsg, msg); - ASSERT(pAction != NULL); - - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - LockObj(pAction); - pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); - pthread_setcancelstate(iCancelStateSave, NULL); - iRet = doActionCallAction(pAction, pMsg); - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - UnlockObj(pAction); - pthread_cleanup_pop(0); /* remove mutex cleanup handler */ - pthread_setcancelstate(iCancelStateSave, NULL); + if(pAction->pQueue->qType == QUEUETYPE_DIRECT) + iRet = qqueueEnqObjDirect(pAction->pQueue, (void*) MsgAddRef(pMsg)); + else + iRet = qqueueEnqObj(pAction->pQueue, pMsg->flowCtlType, (void*) MsgAddRef(pMsg)); RETiRet; } -#pragma GCC diagnostic warning "-Wempty-body" -/* add our cfsysline handlers - * rgerhards, 2008-01-28 + +/* This submits the message to the action queue in case where we need to handle + * bWriteAllMarkMessage == FALSE only. Note that we use a non-blocking CAS loop + * for the synchronization. Here, we just modify the filter condition to be false when + * a mark message must not be written. However, in this case we must save the previous + * filter as we may need it in the next action (potential future optimization: check if this is + * the last action TODO). + * rgerhards, 2010-06-08 */ -rsRetVal -actionAddCfSysLineHdrl(void) +static rsRetVal +doSubmitToActionQNotAllMarkBatch(action_t *pAction, batch_t *pBatch) { + time_t now = 0; + time_t lastAct; + int i; + int bProcessMarkMsgs; + int bModifiedFilter; + sbool FilterSave[128]; + sbool *pFilterSave; DEFiRet; - CHKiRet(regCfSysLineHdlr((uchar *)"actionname", 0, eCmdHdlrGetWord, NULL, &pszActionName, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszActionQFName, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesize", 0, eCmdHdlrInt, NULL, &iActionQueueSize, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iActionQueMaxDiskSpace, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iActionQHighWtrMark, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuelowwatermark", 0, eCmdHdlrInt, NULL, &iActionQLowWtrMark, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardmark", 0, eCmdHdlrInt, NULL, &iActionQDiscardMark, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardseverity", 0, eCmdHdlrInt, NULL, &iActionQDiscardSeverity, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iActionQPersistUpdCnt, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bActionQSyncQeueFiles, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetype", 0, eCmdHdlrGetWord, setActionQueType, NULL, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iActionQueueNumWorkers, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoQShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutactioncompletion", 0, eCmdHdlrInt, NULL, &iActionQtoActShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutenqueue", 0, eCmdHdlrInt, NULL, &iActionQtoEnq, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkertimeoutthreadshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoWrkShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreadminimummessages", 0, eCmdHdlrInt, NULL, &iActionQWrkMinMsgs, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxfilesize", 0, eCmdHdlrSize, NULL, &iActionQueMaxFileSize, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesaveonshutdown", 0, eCmdHdlrBinary, NULL, &bActionQSaveOnShutdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeueslowdown", 0, eCmdHdlrInt, NULL, &iActionQueueDeqSlowdown, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimebegin", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinFromHr, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinToHr, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtime", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccur, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtimetimeout", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccurTO, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgcontainsoriginalmsg", 0, eCmdHdlrBinary, NULL, &bActionRepMsgHasMsg, NULL)); + if(batchNumMsgs(pBatch) <= (int) (sizeof(FilterSave)/sizeof(sbool))) { + pFilterSave = FilterSave; + } else { + CHKmalloc(pFilterSave = malloc(batchNumMsgs(pBatch) * sizeof(sbool))); + } + + bModifiedFilter = 0; + for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { + pFilterSave[i] = pBatch->pElem[i].bFilterOK; + if(((msg_t*)(pBatch->pElem[i].pUsrp))->msgFlags & MARK) { + /* check if we need to write or not */ + if(now == 0) { + now = datetime.GetTime(NULL); /* good time call - the only one done */ + /* CAS loop, we write back a bit early, but that's OK... */ + /* we use reception time, not dequeue time - this is considered more appropriate and + * also faster ;) -- rgerhards, 2008-09-17 */ + do { + lastAct = pAction->f_time; + if((now - lastAct) < MarkInterval / 2) { + DBGPRINTF("action was recently called, ignoring mark message\n"); + bProcessMarkMsgs = 0; + } else { + bProcessMarkMsgs = 1; + } + } while(ATOMIC_CAS(&pAction->f_time, lastAct, + ((msg_t*)(pBatch->pElem[i].pUsrp))->ttGenTime, &pAction->mutCAS) == 0); + } + if(bProcessMarkMsgs) { + pBatch->pElem[i].bFilterOK = 0; + bModifiedFilter = 1; + } + } + } + DBGPRINTF("Called action(NotAllMark), logging to %s\n", module.GetStateName(pAction->pMod)); + + iRet = doSubmitToActionQBatch(pAction, pBatch); + + if(bModifiedFilter) { + /* in this case, we need to restore previous state */ + for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { + pBatch->pElem[i].bFilterOK = pFilterSave[i]; + } + } + finalize_it: + if(pFilterSave != FilterSave) + free(pFilterSave); + + RETiRet; +} + + +/* This submits the message to the action queue in case we do NOT need to handle repeat + * message processing. That case permits us to gain lots of freedom during processing + * and thus speed. + * rgerhards, 2010-06-08 + */ +static rsRetVal +doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch) +{ + int i; + DEFiRet; + + DBGPRINTF("Called action(Batch), logging to %s\n", module.GetStateName(pAction->pMod)); + if(pAction->pQueue->qType == QUEUETYPE_DIRECT) + iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch); + else { /* in this case, we do single submits to the queue. + * TODO: optimize this, we may do at least a multi-submit! + */ + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if(pBatch->pElem[i].bFilterOK) { + doSubmitToActionQ(pAction, (msg_t*)(pBatch->pElem[i].pUsrp)); + } + } + } + RETiRet; } + +/* Helper to submit a batch of actions to the engine. Note that we have rather + * complicated processing here, so we need to do this one message after another. + * rgerhards, 2010-06-23 + */ +static inline rsRetVal +helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch) +{ + int i; + DEFiRet; + + DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod)); + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if(pBatch->pElem[i].bFilterOK) { + doActionCallAction(pAction, (msg_t*)(pBatch->pElem[i].pUsrp)); + } + } + + RETiRet; +} + +/* Call configured action, most complex case with all features supported (and thus slow). + * rgerhards, 2010-06-08 + */ +#pragma GCC diagnostic ignored "-Wempty-body" +static rsRetVal +doSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch) +{ + DEFiRet; + + LockObj(pAction); + pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); + iRet = helperSubmitToActionQComplexBatch(pAction, pBatch); + UnlockObj(pAction); + pthread_cleanup_pop(0); /* remove mutex cleanup handler */ + + RETiRet; +} +#pragma GCC diagnostic warning "-Wempty-body" + /* add an Action to the current selector * The pOMSR is freed, as it is not needed after this function. * Note: this function pulls global data that specifies action config state. @@ -917,6 +1510,8 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques pAction->pModData = pModData; pAction->pszName = pszActionName; pszActionName = NULL; /* free again! */ + pAction->bWriteAllMarkMsgs = bActionWriteAllMarkMsgs; + bActionWriteAllMarkMsgs = FALSE; /* reset */ pAction->bExecWhenPrevSusp = bActExecWhenPrevSusp; pAction->iSecsExecOnceInterval = iActExecOnceInterval; pAction->iExecEveryNthOccur = iActExecEveryNthOccur; @@ -935,15 +1530,12 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques if(pAction->iNumTpls > 0) { /* we first need to create the template pointer array */ CHKmalloc(pAction->ppTpl = (struct template **)calloc(pAction->iNumTpls, sizeof(struct template *))); - CHKmalloc(pAction->ppMsgs = (uchar**) calloc(pAction->iNumTpls, sizeof(uchar *))); - CHKmalloc(pAction->lenMsgs = (size_t*) calloc(pAction->iNumTpls, sizeof(size_t))); } for(i = 0 ; i < pAction->iNumTpls ; ++i) { CHKiRet(OMSRgetEntry(pOMSR, i, &pTplName, &iTplOpts)); - /* Ok, we got everything, so it now is time to look up the - * template (Hint: templates MUST be defined before they are - * used!) + /* Ok, we got everything, so it now is time to look up the template + * (Hint: templates MUST be defined before they are used!) */ if((pAction->ppTpl[i] = tplFind((char*)pTplName, strlen((char*)pTplName))) == NULL) { snprintf(errMsg, sizeof(errMsg) / sizeof(char), @@ -965,6 +1557,8 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques /* set parameter-passing mode */ if(iTplOpts & OMSR_TPL_AS_ARRAY) { pAction->eParamPassing = ACT_ARRAY_PASSING; + } else if(iTplOpts & OMSR_TPL_AS_MSG) { + pAction->eParamPassing = ACT_MSG_PASSING; } else { pAction->eParamPassing = ACT_STRING_PASSING; } @@ -981,10 +1575,10 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques DBGPRINTF("module is incompatible with RepeatedMsgReduction - turned off\n"); pAction->f_ReduceRepeated = 0; } - pAction->bEnabled = 1; /* action is enabled */ + pAction->eState = ACT_STATE_RDY; /* action is enabled */ if(bSuspended) - actionSuspend(pAction, time(NULL)); /* "good" time call, only during init and unavoidable */ + actionSuspend(pAction, datetime.GetTime(NULL)); /* "good" time call, only during init and unavoidable */ CHKiRet(actionConstructFinalize(pAction)); @@ -1006,6 +1600,17 @@ finalize_it: } +/* Reset config variables to default values. + * rgerhards, 2009-11-12 + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + iActExecOnceInterval = 0; + return RS_RET_OK; +} + + /* TODO: we are not yet a real object, the ClassInit here just looks like it is.. */ rsRetVal actionClassInit(void) @@ -1017,6 +1622,36 @@ rsRetVal actionClassInit(void) CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionname", 0, eCmdHdlrGetWord, NULL, &pszActionName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszActionQFName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesize", 0, eCmdHdlrInt, NULL, &iActionQueueSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionwriteallmarkmessages", 0, eCmdHdlrBinary, NULL, &bActionWriteAllMarkMsgs, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuebatchsize", 0, eCmdHdlrInt, NULL, &iActionQueueDeqBatchSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iActionQueMaxDiskSpace, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iActionQHighWtrMark, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuelowwatermark", 0, eCmdHdlrInt, NULL, &iActionQLowWtrMark, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardmark", 0, eCmdHdlrInt, NULL, &iActionQDiscardMark, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardseverity", 0, eCmdHdlrInt, NULL, &iActionQDiscardSeverity, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iActionQPersistUpdCnt, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bActionQSyncQeueFiles, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetype", 0, eCmdHdlrGetWord, setActionQueType, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iActionQueueNumWorkers, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoQShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutactioncompletion", 0, eCmdHdlrInt, NULL, &iActionQtoActShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutenqueue", 0, eCmdHdlrInt, NULL, &iActionQtoEnq, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkertimeoutthreadshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoWrkShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreadminimummessages", 0, eCmdHdlrInt, NULL, &iActionQWrkMinMsgs, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxfilesize", 0, eCmdHdlrSize, NULL, &iActionQueMaxFileSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesaveonshutdown", 0, eCmdHdlrBinary, NULL, &bActionQSaveOnShutdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeueslowdown", 0, eCmdHdlrInt, NULL, &iActionQueueDeqSlowdown, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimebegin", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinFromHr, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinToHr, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtime", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccur, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtimetimeout", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccurTO, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyonceeveryinterval", 0, eCmdHdlrInt, NULL, &iActExecOnceInterval, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgcontainsoriginalmsg", 0, eCmdHdlrBinary, NULL, &bActionRepMsgHasMsg, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); + finalize_it: RETiRet; } @@ -36,18 +36,30 @@ extern int glbliActionResumeRetryCount; +typedef enum { + ACT_STATE_DIED = 0, /* action permanently failed and now disabled - MUST BE ZEO! */ + ACT_STATE_RDY = 1, /* action ready, waiting for new transaction */ + ACT_STATE_ITX = 2, /* transaction active, waiting for new data or commit */ + ACT_STATE_COMM = 3, /* transaction finished (a transient state) */ + ACT_STATE_RTRY = 4, /* failure occured, trying to restablish ready state */ + ACT_STATE_SUSP = 5 /* suspended due to failure (return fail until timeout expired) */ +} action_state_t; + /* the following struct defines the action object data structure */ +typedef struct action_s action_t; struct action_s { time_t f_time; /* used for "message repeated n times" - be careful, old, old code */ time_t tActNow; /* the current time for an action execution. Initially set to -1 and populated on an as-needed basis. This is a performance optimization. */ time_t tLastExec; /* time this action was last executed */ - bool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ + sbool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ + sbool bWriteAllMarkMsgs;/* should all mark msgs be written (not matter how recent the action was executed)? */ int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ - short bEnabled; /* is the related action enabled (1) or disabled (0)? */ - bool bSuspended; /* is the related action temporarily suspended? */ + action_state_t eState; /* current state of action */ + sbool bHadAutoCommit; /* did an auto-commit happen during doAction()? */ time_t ttResumeRtry; /* when is it time to retry the resume? */ + int iResumeOKinRow; /* number of times in a row that resume said OK with an immediate failure following */ int iResumeInterval;/* resume interval for this action */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ int iNbrResRtry; /* number of retries since last suspend */ @@ -57,11 +69,13 @@ struct action_s { time_t tLastOccur; /* time last occurence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ - bool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ + sbool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ - enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1 } + rsRetVal (*submitToActQ)(action_t *, batch_t *);/* function submit message to action queue */ + rsRetVal (*qConstruct)(struct queue_s *pThis); + enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1, ACT_MSG_PASSING } eParamPassing; /* mode of parameter passing to action */ int iNumTpls; /* number of array entries for template element below */ struct template **ppTpl;/* array of template to use - strings must be passed to doAction @@ -74,11 +88,9 @@ struct action_s { SYNC_OBJ_TOOL; /* required for mutex support */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ uchar *pszName; /* action name (for documentation) */ - //uchar **ppMsgs; /* pointer to action-calling parameters (kept in structure to save alloc() time!) */ - void *ppMsgs; /* pointer to action-calling parameters (kept in structure to save alloc() time!) */ - size_t *lenMsgs; /* length of message in ppMsgs */ + int *pbShutdownImmediate;/* to facilitate shutdown, if var is 1, shut down immediately */ + DEF_ATOMIC_HELPER_MUT(mutCAS); }; -typedef struct action_s action_t; /* function prototypes @@ -86,11 +98,9 @@ typedef struct action_s action_t; rsRetVal actionConstruct(action_t **ppThis); rsRetVal actionConstructFinalize(action_t *pThis); rsRetVal actionDestruct(action_t *pThis); -rsRetVal actionAddCfSysLineHdrl(void); rsRetVal actionDbgPrint(action_t *pThis); rsRetVal actionSetGlobalResumeInterval(int iNewVal); rsRetVal actionDoAction(action_t *pAction); -rsRetVal actionCallAction(action_t *pAction, msg_t *pMsg); rsRetVal actionWriteToAction(action_t *pAction); rsRetVal actionCallHUPHdlr(action_t *pAction); rsRetVal actionClassInit(void); @@ -19,7 +19,7 @@ fi (cd $srcdir && autoreconf --verbose --force --install) || exit 1 -conf_flags="--enable-shave --cache-file=config.cache" +conf_flags="--cache-file=config.cache" if test x$NOCONFIGURE = x; then echo Running $srcdir/configure $conf_flags "$@" ... diff --git a/configure.ac b/configure.ac index 27c8d69f..9e0aad5c 100644 --- a/configure.ac +++ b/configure.ac @@ -2,8 +2,11 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.7.2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.5.7],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE + +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) + AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) @@ -12,7 +15,7 @@ AC_GNU_SOURCE # check for Java compiler AC_CHECK_PROG(HAVE_JAVAC, [javac], [yes]) -if test x"$HAVE_JAVAC" = x"yes"; then +if test x"$HAVE_JAVAC" = x""; then AC_MSG_WARN([no javac found, disabling features depending on it]) fi @@ -105,7 +108,7 @@ AC_TYPE_SIGNAL AC_FUNC_STAT AC_FUNC_STRERROR_R AC_FUNC_VPRINTF -AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline malloc_trim prctl fdatasync]) +AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r getline malloc_trim prctl epoll_create epoll_create1 fdatasync]) # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) @@ -126,6 +129,7 @@ AC_TRY_COMPILE([ # check for availability of atomic operations RS_ATOMIC_OPERATIONS +RS_ATOMIC_OPERATIONS_64BIT # fall back to POSIX sems for atomic operations (cpu expensive) AC_CHECK_HEADERS([semaphore.h]) @@ -179,6 +183,7 @@ if test "$enable_regexp" = "yes"; then fi + # zlib compression AC_ARG_ENABLE(zlib, [AS_HELP_STRING([--enable-zlib],[Enable zlib compression support @<:@default=yes@:>@])], @@ -236,7 +241,7 @@ AC_ARG_ENABLE(pthreads, ) if test "x$enable_pthreads" = "xno"; then - AC_MSG_ERROR(rsyslog v3 does no longer support single threading mode -- use a previous version for that); + AC_MSG_ERROR(rsyslog v3+ does no longer support single threading mode -- use a previous version for that); fi if test "x$enable_pthreads" != "xno"; then @@ -403,6 +408,21 @@ if test "$enable_valgrind" = "yes"; then fi +# memcheck +AC_ARG_ENABLE(memcheck, + [AS_HELP_STRING([--enable-memcheck],[Enable extended memory check support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_memcheck="yes" ;; + no) enable_memcheck="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-memcheck) ;; + esac], + [enable_memcheck="no"] +) +if test "$enable_memcheck" = "yes"; then + AC_DEFINE(MEMCHECK, 1, [Defined if memcheck support settings are to be enabled (e.g. prevents dlclose()).]) +fi + + # compile diagnostic tools (small helpers usually not needed) AC_ARG_ENABLE(diagtools, [AS_HELP_STRING([--enable-diagtools],[Enable diagnostic tools @<:@default=no@:>@])], @@ -785,6 +805,36 @@ AC_ARG_ENABLE(omprog, AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) +# settings for omudpspoof +AC_ARG_ENABLE(omudpspoof, + [AS_HELP_STRING([--enable-omudpspoof],[Compiles omudpspoof module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omudpspoof="yes" ;; + no) enable_omudpspoof="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omudpspoof) ;; + esac], + [enable_omudpspoof=no] +) + +if test "x$enable_omudpspoof" = "xyes"; then + AC_CHECK_HEADERS( + [libnet.h],, + [AC_MSG_FAILURE([libnet is missing])] + ) + AC_CHECK_LIB( + [net], + [libnet_init], + [UDPSPOOF_CFLAGS="" + UDPSPOOF_LIBS="-lnet" + ], + [AC_MSG_FAILURE([libnet is missing])] + ) +fi +AM_CONDITIONAL(ENABLE_OMUDPSPOOF, test x$enable_omudpspoof = xyes) +AC_SUBST(UDPSPOOF_CFLAGS) +AC_SUBST(UDPSPOOF_LIBS) + + # settings for omstdout AC_ARG_ENABLE(omstdout, [AS_HELP_STRING([--enable-omstdout],[Compiles stdout module @<:@default=no@:>@])], @@ -798,6 +848,80 @@ AC_ARG_ENABLE(omstdout, AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) +# settings for pmlastmsg +AC_ARG_ENABLE(pmlastmsg, + [AS_HELP_STRING([--enable-pmlastmsg],[Compiles lastmsg parser module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_pmlastmsg="yes" ;; + no) enable_pmlastmsg="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmlastmsg) ;; + esac], + [enable_pmlastmsg=no] +) +AM_CONDITIONAL(ENABLE_PMLASTMSG, test x$enable_pmlastmsg = xyes) + + +# settings for pmrfc3164sd +AC_ARG_ENABLE(pmrfc3164sd, + [AS_HELP_STRING([--enable-pmrfc3164sd],[Compiles rfc3164sd parser module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_pmrfc3164sd="yes" ;; + no) enable_pmrfc3164sd="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmrfc3164sd) ;; + esac], + [enable_pmrfc3164sd=no] +) +AM_CONDITIONAL(ENABLE_PMRFC3164SD, test x$enable_pmrfc3164sd = xyes) + + +# settings for omruleset +AC_ARG_ENABLE(omruleset, + [AS_HELP_STRING([--enable-omruleset],[Compiles ruleset forwarding module @<:@default=yes@:>@])], + [case "${enableval}" in + yes) enable_omruleset="yes" ;; + no) enable_omruleset="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omruleset) ;; + esac], + [enable_omruleset=yes] +) +AM_CONDITIONAL(ENABLE_OMRULESET, test x$enable_omruleset = xyes) + + +# settings for omdbalerting +AC_ARG_ENABLE(omdbalerting, + [AS_HELP_STRING([--enable-omdbalerting],[Compiles omdbalerting module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omdbalerting="yes" ;; + no) enable_omdbalerting="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omdbalerting) ;; + esac], + [enable_omdbalerting=no] +) +AM_CONDITIONAL(ENABLE_OMDBALERTING, test x$enable_omdbalerting = xyes) + + + +# building the GUI (mostly for diagnostic reasons) +AC_ARG_ENABLE(gui, + [AS_HELP_STRING([--enable-gui],[Enable GUI programs @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_gui="yes" ;; + no) enable_gui="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-gui) ;; + esac], + [enable_gui=no] +) +if test "x$enable_gui" = "xyes"; then + if test x$HAVE_JAVAC = x; then + AC_MSG_ERROR([GUI components need Java, but Java development system is not installed on this system]) + fi +fi +AM_CONDITIONAL(ENABLE_GUI, test x$enable_gui = xyes) + + +AC_SUBST(RELP_CFLAGS) +AC_SUBST(RELP_LIBS) + # settings for omuxsock AC_ARG_ENABLE(omuxsock, [AS_HELP_STRING([--enable-omuxsock],[Compiles stdout module @<:@default=no@:>@])], @@ -866,12 +990,7 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to search for omtemplate to find everything! -SHAVE_INIT - - AC_CONFIG_FILES([Makefile \ - shave \ - shave-libtool \ runtime/Makefile \ tools/Makefile \ doc/Makefile \ @@ -886,6 +1005,10 @@ AC_CONFIG_FILES([Makefile \ plugins/omtemplate/Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ + plugins/pmrfc3164sd/Makefile \ + plugins/pmlastmsg/Makefile \ + plugins/omruleset/Makefile \ + plugins/omdbalerting/Makefile \ plugins/omuxsock/Makefile \ plugins/imfile/Makefile \ plugins/imsolaris/Makefile \ @@ -901,20 +1024,22 @@ AC_CONFIG_FILES([Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ + plugins/omudpspoof/Makefile \ plugins/cust1/Makefile \ + java/Makefile \ tests/Makefile]) AC_OUTPUT echo "****************************************************" echo "rsyslog will be compiled with the following settings:" echo -echo " Multithreading support enabled: $enable_pthreads" echo " Large file support enabled: $enable_largefile" echo " Networking support enabled: $enable_inet" echo " Regular expressions support enabled: $enable_regexp" echo " Zlib compression support enabled: $enable_zlib" echo " rsyslog runtime will be built: $enable_rsyslogrt" echo " rsyslogd will be built: $enable_rsyslogd" +echo " GUI components will be built: $enable_gui" echo " custom module 1 will be built: $enable_cust1" echo " Unlimited select() support enabled: $enable_unlimited_select" echo @@ -930,9 +1055,16 @@ echo "---{ output plugins }---" echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" +echo " omruleset module will be compiled: $enable_omruleset" +echo " omdbalerting module will be compiled: $enable_omdbalerting" +echo " omudpspoof module will be compiled: $enable_omudpspoof" echo " omuxsock module will be compiled: $enable_omuxsock" echo " output template module will be compiled: $enable_omtemplate" echo +echo "---{ parser modules }---" +echo " pmrfc3164sd module will be compiled: $enable_pmrfc3164sd" +echo " pmlastmsg module will be compiled: $enable_pmlastmsg" +echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" echo " libdbi support enabled: $enable_libdbi" @@ -951,5 +1083,6 @@ echo " Extended Testbench enabled: $enable_extended_tests" echo " Debug mode enabled: $enable_debug" echo " Runtime Instrumentation enabled: $enable_rtinst" echo " Diagnostic tools enabled: $enable_diagtools" +echo " Enhanced memory checking enabled: $enable_memcheck" echo " Valgrind support settings enabled: $enable_valgrind" echo @@ -31,10 +31,8 @@ rsRetVal multiSubmitMsg(multi_submit_t *pMultiSub); rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime); -int parseRFCSyslogMsg(msg_t *pMsg, int flags); -int parseLegacySyslogMsg(msg_t *pMsg, int flags); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ -char* getFIOPName(unsigned iFIOP); +rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName); /* Intervals at which we flush out "message repeated" messages, * in seconds after previous message is logged. After each flush, @@ -42,32 +40,13 @@ char* getFIOPName(unsigned iFIOP); * TODO: move this to action object! Only action.c and syslogd.c use it. */ extern int bActExecWhenPrevSusp; -extern int iActExecOnceInterval; extern int MarkInterval; extern int repeatinterval[2]; extern int bReduceRepeatMsgs; +extern qqueue_t *pMsgQueue; /* the main message queue */ #define MAXREPEAT ((int)((sizeof(repeatinterval) / sizeof(repeatinterval[0])) - 1)) #define REPEATTIME(f) ((f)->f_time + repeatinterval[(f)->f_repeatcount]) #define BACKOFF(f) { if (++(f)->f_repeatcount > MAXREPEAT) \ (f)->f_repeatcount = MAXREPEAT; \ } -extern int bDropTrailingLF; -extern uchar cCCEscapeChar; -extern int bEscapeCCOnRcv; -extern int bEscapeTab; -#ifdef USE_NETZIP -/* config param: minimum message size to try compression. The smaller - * the message, the less likely is any compression gain. We check for - * gain before we submit the message. But to do so we still need to - * do the (costly) compress() call. The following setting sets a size - * for which no call to compress() is done at all. This may result in - * a few more bytes being transmited but better overall performance. - * Note: I have not yet checked the minimum UDP packet size. It might be - * that we do not save anything by compressing very small messages, because - * UDP might need to pad ;) - * rgerhards, 2006-11-30 - */ -#define MIN_SIZE_FOR_COMPRESS 60 -#endif - #endif /* #ifndef DIRTY_H_INCLUDED */ diff --git a/doc/Makefile.am b/doc/Makefile.am index 72954e9c..d4df740a 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -31,6 +31,9 @@ html_files = \ version_naming.html \ contributors.html \ dev_queue.html \ + omstdout.html \ + omudpspoof.html \ + omruleset.html \ omsnmp.html \ ommysql.html \ omoracle.html \ @@ -43,6 +46,7 @@ html_files = \ imsolaris.html \ imuxsock.html \ imklog.html \ + pmlastmsg.html \ queues.html \ src/queueWorkerLogic.dia \ queueWorkerLogic.jpg \ @@ -65,6 +69,7 @@ html_files = \ rsconf1_actionresumeinterval.html \ rsconf1_allowedsender.html \ rsconf1_controlcharacterescapeprefix.html \ + rsconf1_escape8bitcharsonreceive.html \ rsconf1_debugprintcfsyslinehandlerlist.html \ rsconf1_debugprintmodulelist.html \ rsconf1_debugprinttemplatelist.html \ @@ -90,9 +95,12 @@ html_files = \ rsconf1_moddir.html \ rsconf1_repeatedmsgreduction.html \ rsconf1_resetconfigvariables.html \ + rsconf1_rulesetcreatemainqueue.html \ rsconf1_umask.html \ + rsconf1_rulesetparser.html \ v3compatibility.html \ v4compatibility.html \ + v5compatibility.html \ im3195.html \ netstream.html \ ns_gtls.html \ @@ -120,6 +128,7 @@ html_files = \ grfx_files = \ rsyslog_confgraph_complex.png\ rsyslog_confgraph_std.png \ + module_workflow.png \ direct_queue0.png \ direct_queue1.png \ direct_queue2.png \ @@ -130,6 +139,8 @@ grfx_files = \ dataflow.png \ queue_analogy_tv.png \ gssapi.png \ + rfc5424layers.png \ + src/rfc5424layers.dia \ rsyslog-vers.png EXTRA_DIST = $(html_files) $(grfx_files) diff --git a/doc/action-call.dot b/doc/action-call.dot new file mode 100644 index 00000000..86c6834d --- /dev/null +++ b/doc/action-call.dot @@ -0,0 +1,33 @@ +// This file is part of rsyslog. +// +// rsyslog action call state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot action-call.dot -Tpng >action-call.png + +digraph G { + label="\n\nrsyslog message states during action processing\nhttp://www.rsyslog.com"; + //fontsize=20; + + ok [label="ready for processing" color="green"]; + mpf [label="message permanent failure" color="red"]; + tf [label="temporary failure"] + cPen [label="commit pending"]; + com [label="committed" color="red"]; + + tf -> tf [label="retry fails, i < n"]; + tf -> mpf [label="retry fails, i = n"]; + tf -> ok [label="retry succeeds"]; + ok -> com [label="doAction RS_RET_OK"]; + ok -> cPen [label="doAction COMMIT_PENDING"]; + ok -> tf [label="doAction RS_RET_SUSPENDED"]; + ok -> mpf [label="doAction RS_RET_DISABLED"]; + cPen -> com [label="endTransaction RS_RET_OK"]; + cPen -> tf [label="endTransaction _SUSPENDED"]; + + //{rank=same; tf cPen} + {rank=same; com mpf} +} diff --git a/doc/action_state.dot b/doc/action_state.dot new file mode 100644 index 00000000..2f36d8da --- /dev/null +++ b/doc/action_state.dot @@ -0,0 +1,34 @@ +// This file is part of rsyslog. +// +// rsyslog message state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog action transaction states\nhttp://www.rsyslog.com"; + //fontsize=20; + + rdy [label="ready" group="main"]; + itx [label="in Tx" group="main"]; + comm [label="commit"] + rtry [label="retry"] + susp [label="suspended"] + + rdy -> itx [label="transaction begins"] + rdy -> rtry [label="begin tx\nerror"] + itx -> itx [label="success"] + itx -> comm [label="commit\n(caller or auto)"] + itx -> rtry [label="error"] + comm -> rdy [label="success"] + comm -> rtry [label="error"] + rtry -> rdy [label="recovered"] + rtry -> susp [label="could not\nrecover"] + susp -> rtry [label="timeout expired"] + + {rank=same; comm rtry} +} diff --git a/doc/batch_state.dot b/doc/batch_state.dot new file mode 100644 index 00000000..0dd48b47 --- /dev/null +++ b/doc/batch_state.dot @@ -0,0 +1,28 @@ +// This file is part of rsyslog. +// +// rsyslog batch state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog batch states\nhttp://www.rsyslog.com"; + rankdir=LR + + rdy [label="ready"]; + bad [label="message-caused\nfailure"]; + sub [label="submitted"] + disc [label="discarded" color="red"] + + rdy -> sub [label="submitted to action"] + rdy -> bad [label="permanent fail"] + rdy -> disc [label="action requests discarding"] + sub -> rdy [label="next action or\naction-caused failure"] + bad -> rdy [label="next action"] + + //{rank=same; comm rtry } +} diff --git a/doc/build_from_repo.html b/doc/build_from_repo.html index 8d3b20fe..a06863e1 100644 --- a/doc/build_from_repo.html +++ b/doc/build_from_repo.html @@ -43,12 +43,37 @@ you downloaded an official distribution tarball (see the <a href="install.html">rsyslog install guide</a>, starting at step 2, for further details about that). +<h2>Special Compile-Time Options</h2> +<p>On some platforms, compile-time issues occur, like the one shown below: +<p><pre><code> +make[2]: Entering directory `/home/az/RSyslog/rsyslog-5.5.0/tools' + CCLD rsyslogd +rsyslogd-omfile.o: In function `getClockFileAccess': +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +/home/az/RSyslog/rsyslog-5.5.0/tools/omfile.c:91: undefined reference to `__sync_fetch_and_add_8' +</code></pre> +<p>Note that the exact error messages can be different. These type of errors stem down to +atomic instruction support in GCC, which is somewhat depending on the machine architecture it +compiles code for. Very old machines (like the original i386) do not even at all provide support +for these instructions. +<p>The availability of atomic instructions is vital for rsyslog - it can not be built without them. +Consequently, there is a configure check included for them. But under some circumstances, +GCC seems to report they are available, but does not provide implementations for +all of them (at least this is my observation...). The simple cure is to make sure that +GCC generates code for a modern-enough architecture. This, for example, can be done as +follows: +<p><pre><code> +./configure CFLAGS="-march=i586 -mcpu=i686" --enable-imfile ... (whatever you need) +</code></pre> +<p>These settings should resolve the issue . + <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 1.2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/debug.html b/doc/debug.html index 46759986..6aeb7975 100644 --- a/doc/debug.html +++ b/doc/debug.html @@ -138,7 +138,7 @@ instance of rsyslogd can be aborted by pressing ctl-c. <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body> diff --git a/doc/design.tex b/doc/design.tex new file mode 100644 index 00000000..a3ec8f45 --- /dev/null +++ b/doc/design.tex @@ -0,0 +1,887 @@ +\documentclass[a4paper,10pt]{article} +\usepackage{amsmath} +\usepackage{amsfonts} +\usepackage{amssymb} +\usepackage{graphicx} +\usepackage{listings} +\usepackage{algorithm,algorithmic} +\usepackage{float} + +\pagestyle{headings} + +\newcommand{\IN}{\mathbb{N}} +\newcommand{\MM}{\mathcal{M}} +\newcommand{\QQ}{\mathcal{Q}} +\newcommand{\AAA}{\mathcal{A}} +\title{Rsyslog Design and Internals} +\author{Rainer Gerhards\\ +rgerhards@adiscon.com} + +\begin{document} + +\maketitle + +\begin{abstract} +This paper describes rsyslog design and internals. It is created to facilitate a discussion about the implementation of "batched queue processing". As such, it does not describe the full design of rsyslog but rather those elements that are relevant to queues. However, the document may be expanded in the future. This is work in progress and should be considered with care. It is NOT updated during all phases of development. +\end{abstract} + +\tableofcontents + +\section{Preliminaries} +\subsection{On the Use of English} +\begin{quotation} +\begin{flushright} +I ventured to write this book in English because ... \\ +it will be more easily read in poor English, \\ +than in good German by 90\% of my intended readers. \\ +--- HANS J. STETTER, Analysis of Discretization Methods for \\ +Ordinary Differential Equations (1973) +\end{flushright} +\end{quotation} + +There is not much I could add to Mr. Stetter's thought, except, maybe, that the number to quote probably tends more to 99\% in this case than to the 90\% Mr. Stetter notes. So please pardon those errors in language use that I have not yet been able to fix or even see. Suggestions for corrections and improvements are always welcome. +\subsection{Notational Conventions} +In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathcal{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in caligraphic letters ($\mathcal{O}$). Often, objects $O_i, i \in \IN, i \le |\mathcal{O}|$ partition $\mathcal{O}$, but this is not necessarily the case. + +\subsection{Definitions} +\subsubsection{Sudden Fatal Failure} +As sudden fatal failure is one that occurs at some instant and causes Complete loss of processing capabilities. The two major cases are a sudden power loss or a ``kill -9'' of the process. There are more exotic cases, too, like disasters. + +One may argue that it is possible to protect against many sudden fatal failure cases. For example, using an uninterruptable power supply (UPS) will prevent a sudden power loss. While this is true in most cases, it does not hold if looked very closely: in the case of the UPS, for example, a failure in the UPS itself may cause a sudden power loss, which can not be mitigated. Well, actually there can be several layers of mitigation, but always one more potential failure scenario remains. So it is not possible to totally solve the issue. + +The concept of ``sudden fatal failure'' now covers all these rest risk that result in termiantion of rsyslogd without the ability execute any code before this happens. This is a very important concept in regard to audit-gradeness. + +\subsubsection{Audit Grade} +In the context of this document, ``audit grade'' means that a subsystem never loses a message that it has taken responsibility for, not even in cases of sudden fatal failures. The only limit in this restriction is that a subsystem does not guarantee message survival if the subsytem at large is being destroyed (e.g. during a disaster) or some of its components are not of audit-grade. This draws a fine limitation on the audit-grade of a subsystem. + +For example, the rsyslog queue subsystem receives messages and acknowledges them to the submitter (e.g. an input), when they have been enqueued in the storage system. If the queue system is configured to provide audit-grade operation\footnote{Audit-grade queue operation is considerably slower than regular operations, as such this mode is not enabled by default. Most installations will never need a completely audit-grade queue}, the queue relies on the storage subsystem to work properly. If, for example, a disk read error occurs, the message may no longer be readable from the disk and as such is lost. The root cause here is that the disk subsystem was not of audit grade, because it otherwise would not have lost the message. So in this case the queue code is of audit grade, but the one of its components, the disk subsytem, was not. So the overall system is not of audit grade. + +To simplify talking about the audit-gradness of several subsytems, we assume that all of their subsystems are also of audit grade. In an actual deployment, however, this means the the system designer must carefully select audit-grade subsystems. Overlooking a single non-audit-grade component will make the whole system of not audit grade quality. + +Please note that it can be rather tricky to ensure a complete system is of audit grade. A border case is main memory integrity. Even with error-correcting memory, there may situations arise where a memory error occurs (probably due to a very unlikely series of well-hitting cosmic rays) that is unrecoverable. At this point, system integrity is at risk. The only real solution is to immediately shut down the system and restart it (without giving any process a chance to execute). Note, however, that in an extreme view, an operating system routine that does so can also be considered dangerous, as memory in use by this routine might be affected by the malfunction. We could extend this scenario and further complicate it, but that goes beyond the scope of this paper. The example was primarily meant to show how subtle audit-grade reliability is. + +In rsyslog, we currently use a slightly \marginpar{duplication\\permitted}relaxed consistency condition for message integrity inside an audit-grade subsystem. While we do not accept message loss, we permit slight message \emph{duplication}, but only in exceptional cases. This is permitted because, with proper message generation, the dulication problem can be easily fixed at the end-to-end layer. For example, the original sender can include a UUID, which can be used to sort out duplicates at the final destination. Insisting on not allowing duplication complicates matters and is often impossible with today's logging protocols. So, for the time being, we aim at this relaxed criteria, which is hard enough to achive. After we have achieved that goal, we may further try to solve the duplicaton problem. Some hooks already exist. But we do not guarantee such an effort will be made any time soon. + +\section{Overall Design} +From a high-level prespective, rsyslogd is ``just'' a high-performance message router. It accepts messages from various sources, applies user-configured filters to them, and routes potentially transformed messages to destinations based on these filters. +\section{Objects} +\subsection{Plugins} +Plugins provide code potentially written by a third party to extend rsyslog. + +Conceptually, a plugin is a tuple of callable functions $(\phi_1, \phi_2, \ldots)$ which implement an interface. There are three different types of plugins: input, output and library. The plugin type denotes the primary interface implemented by the plugin. Additional interfaces may be implemented\footnote{This is not yet done in plugins, but is possible and assumed to be done at a later point in time}. + +In the context of this paper, the output plugin interface is most important. It implements three entry points: + +\paragraph{doAction()} +is used to submit messages to the output plugin. The entry point may or may not commit the messages to their ultimate destination. + +\paragraph{beginTransaction()} +is used to inform the plugin that a new transaction begins. It must prepare for processing. + +\paragraph{endTransaction()} +is indicated that the upper layer \emph{needs} to close the transaction. If there is any uncommited data left, it must be commited or rolled back. + +Every instance of an output plugin is guaranteed \emph{not} to be called concurrently by multiple threads. Further, no context switch will happen between calls to $doAction()$ and $endTransaction()$. + +\subsection{State Sets} +Several object have associated state based on a specific state set. These state sets are described together with the objects. + +As a general rule, individual state is associated with all instances $o$ of a class of objects. This state is called the object's \marginpar{state component} \emph{state component} $s$. If we want to obtain an object's state, we write $S(o)$. Please note that $S(o)$ is only defined for those objects that have a state component. + +\subsection{Messages} +A message $m$ represents a a single syslog message inside the system. It is a tuple of attributes. Some of these attributes directly orginate from the message content, some others are meta-information taken from the context. For example, there is an meta-attribute ``time of reception'' which conveys when the message was received by rsyslog's input subsystem. We do not list attributes here, as there are many and it is not of importance which exactly they are. + +The set $\MM$ is composed of all messages that exist at a given time inside rsyslog. + +\subsection{Queue} +A queue +$$Q = (C, \Phi, M)$$ +is a triplet of a set of configuration parameters $C$, a set of callbacks $\Phi$ and a set of messages $M \subseteq \MM$. + +If we need to obtain the set of message from a queue, we write $M(Q)$. The elements of the set of configuration parameters are written as $C_{param}$ where $param$ is an abbreviation of the parameter's meaning. To obtain a specific parameter from a queue, we write $C_{param}(Q)$. The most important elements of $C$ are: + +\paragraph{$C_{type}$} which denotes the queue implementation type. Most importantly, this selects from a set of queue drivers (for example disk-only or in-memory driver), which affects the basic operation of the queue instance. + +\paragraph{$C_{mMsg}$} which denotes the upper bound on the cardinality of $M$. + +\paragraph{$C_{mBatch}$} which denotes the upper bound of the cardinality of message batches created for this queue. + +Be $\QQ = \{Q_m, Q_1, Q_2, \ldots, Q_{|\AAA|}\}$ the set of all queues that exist inside rsyslog after the configuration file has been processed, with $|\QQ| = |\AAA| + 1$. + +Then +$$M_0 = \MM \setminus \bigcup_{i=1}^{|\QQ|} Q_i(M)$$ +\marginpar{at-risk-set}is the set of non-queued messages. The messages have either never been enqueued or have been dequeued but not finally been processed. This set represents the messages that may potentially be lost during an unclean shutdown of rsyslogd. This is why I call this set the ``\emph{at-risk-set}''. + + +\subsection{Batches} +A batch represents multiple processable messages. It is a unit of processing inside rsyslog's output system. Batches are used to dequeue a number of messages from a queue and then submit them to the lower action layer. Batches are natural \emph{transaction boundaries}, in the sense that multiple output transactions may be done on the messages inside a batch, but each transaction must end at the end of the batch. A batch is always associated to a specific queue $Q$. + +A batch +$$B = (b_1, b_2, \ldots, b_n )$$ +is a $n$-tuple of \marginpar{processable\\message}processable messages +$$b = (m, s)$$ +which are an ordered pair of a message $m$ and an associated processing state $s$. To denote the $n$-th message inside the batch, we write $m(b_n)$, to denote the status component of the $n$-th message, we write $S(b_n)$. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{batch_state.jpeg} +\end{center} +\caption{batch message processing states} +\label{fig_batchmsg_states} +\end{figure} + +The state set for the processing states is defined as follows: +$$ +S_B = \{ rdy, bad, sub, disc \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready for processing\\ + bad & this message triggered an unrecoverable failure in action\\ + & processing and must not be resubmitted to this action\\ + sub & message submitted for processsing, result yet unknown \\ + disc & action sucessfully processed, but must not be submitted \\ + & to any further action in action unit \\\hline +\end{tabular} +\end{center} +The associated state diagram is shown in figure \ref{fig_batchmsg_states} on page \pageref{fig_batchmsg_states}. + +Batch sizes vary. The actual cardinality is a function of the cardinality of $M(Q)$ at the time of batch creation and the queue configuration: + +$$1 \leq |B| \leq \max(C_{mBatch}(Q), |M(Q)|)$$ + +\subsection{Action Unit} +An action unit +$$u = (f, a_1, \ldots, a_n), a_i \in \AAA \text{ for } i \in \IN, i \le n$$ +is a tuple consisting of a filter function $f$ and $n \in \IN$ actions. \emph{Does rsyslog still support nonsense action units with $n=0$? - check!} + +\subsection{Action} +An action +$$a = (a_C, a_\psi)$$ +is an ordered pair of a tuple of configuration attributes $a_C$, and a tuple of processing functions $a_\psi$. Be the set $\AAA$ composed of all actions that exist in rsyslog after the configuration file has been processed. + + +\section{Processing} +\subsection{Object States} +Various objects keep state. Some of these objects, like messages, batches and actions seem to share state. However, thinking about shared state leads to very complex setup. As such, state is modelled for each object $o$ individually. Instead, the state function $S_O(o)$ can be used to obtain an obtain an individual objects state. That state can be used to modify the state diagrams of the other objects with which relationships exist. + +\subsubsection{Actions} +Actions are provided by output plugins. An action enables the engine to write messages to some destination. It is important to note that ``destination'' is a very broad abstraction. A destination may be a file inside a local or remote file system, a database table or a remote syslog server in another network. + +Actions are transactional in the following sense: more than one message can be submitted to an action. The action does not necessarily process the submitted messages unless the caller ends the transaction. However, the action itself may also end the transaction and notify the caller. This is \emph{not} considered an error condition and \emph{must} be handled gracefully by the caller. If a transaction aborts, the caller \emph{must} assume that none of the elements submitted since the begin of transaction have been processed. The action will try to backout anything that was already processed at the time the transaction failed. However, not all outputs work on actually transactional destination. As such, an action is permitted not to backout incomplete interim results. As such, after a transaction abort, some message duplication may occur. We call this the \emph{relaxed integrity condition} for actions. + +An output transaction is started by calling \emph{beginTransaction()} either explicitely or implicitely by a call to \emph{doAction()} without calling \emph{beginTransaction()} before. Then, one or more calls to \emph{doAction()} follow. When the caller intends to finish the transaction, it calls \emph{endTransaction()}. However, the transaction may also be terminated from the action itself in response to a \emph{doAction()} call. + +Mathematically, an action transaction builds a totally ordered set of uncommitted messages $M_u$. The order relation is defined over the sequence in which messages are being provided to \emph{doAction()}. At any time a commit is attempted, the full set $M_u$ is committed and may either succeeed completely or not at all (in the sense of the relaxed integrity condition described above). + +A commit is attempted when +\begin{enumerate} +\item the caller decides to call \emph{endTransaction()} +\item or earlier if the action decides it needs to commit now (e.g. because of buffers filling up). +\end{enumerate} + +In the seconds case, the action may decide to commit all message but the current one or all (this is depending on action logic). So if the action decideds to commit a transaction before the caller calls \emph{endTransaction()}, a set of commited messages $M_c$ is build and $M_u$ is modified. Be $n$ the $n$-th iterated \emph{doAction()} call and $m_n$ the current message of this call, then the sets are build as follows: + +\begin{algorithm} +%\caption{} +\begin{algorithmic} +\IF{action commits $m_n$} + \STATE $M_c = M_u \cup m_n$ + \STATE $M_u = \emptyset$ +\ELSE + \STATE $M_c = M_u$ + \STATE $M_u = \{ m_n\}$ +\ENDIF +\end{algorithmic} +\end{algorithm} + +In other words, if anything is committed early, it is always the full set $M_u$, with or without the current message. The caller needs to know which messages are already commited. As \emph{doAction()} finishes one transaction and starts a new one in a single call, we can not use action state the let the caller know this happened. So we use our above finding and just convey back if the transacton is still continuing or the current message or all others before it were committed. The caller must then act accordingly. Please note that when an error happens, the whole transaction must still be considered failed. As such, ``partial commit'' states need not to be mixed with failure states. + +Please note that the above method leaves a small potential issue unaddressed: if the action does an early commit of $M_u \setminus m_n$, an error happens when adding $m_n$ to the new $M_u$ (like running out of resources), the action would need to convey both the successful transaction as well as the failure state. This is not possible with the current interface. We could use callbacks to provide such notification, but this complicates the code. So, if that situaton arises, the action must temporarily buffer the error condition and convey it as part of either the next \emph{doAction()} call or during \emph{endTransation()} processing. This can be done, for example, by advancing its internal state accordingly. + +The state set for a actions is defined as follows: +$$ +S_A = \{ rdy, itx, comm, rtry, susp, died \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready, waiting for transaction begin\\ + itx & in transaction, accept more data \\ + comm & transaction finished \\ + rtry & action failed but may be able to recover \\ + susp & action currently defunctional until timeout expires \\ + died & unrecoverable error condition occured, no longer usable \\\hline +\end{tabular} +\end{center} + +In the associated state diagram in figure \ref{fig_action_states}, we do not include the \emph{died} state, because it is entered whenever a totally unrecoverable error state may occur. This is a very exceptional incident (which most output plugins do not even support), so we have kept the diagram simple. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.5]{action_state.jpeg} +\end{center} +\caption{Action State Diagram} +\label{fig_action_states} +\end{figure} + +\emph{Note well} that the state diagram describes the action state. It does \emph{not} describe the transaction state. While action- and transaction state are closely related to each other, they are different entities. + +The return code of \emph{doAction()} and \emph{endTransaction()} is used to convey the transaction state. As such, it is a function of the actions's current state after processing the request. The mapping is as shown below: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Return Code (RS\_RET\_\ldots)\\\hline + rdy & OK \\ + itx & COMMITTED (if there was an auto-commit without $m_n$)\\ + & DEFER\_COMMIT (if there was no auto-commit)\\ + comm & internal state, not to be exposed to upper layer \\ + rtry & SUSPENDED \emph{(new code needed)} \\ + susp & SUSPENDED \\ + died & DISABLED \\\hline +\end{tabular} +\end{center} + +For the rest of this document, let's assume there is a function \emph{getReturnCode()} that implements this mapping. + +It is important to think about how retries are handled. There is a user-configured per-action upper number of retries $C_r$ and retry interval $C_i$. In \emph{rsyslog v3}, there is no concept of output transactions. As such, only single messages are processed. When a temporary action failure occurs, the action is re-tried $C_r$ times, where the action processing thread is waiting in a \emph{sleep()} $C_i$ operating system API call\footnote{a suitable API is used, not \emph{sleep()} itself}. If the action succeeds during the retry processing, everything continues as usual. If it does not succeed, two things happen: +\begin{itemize} +\item the message is flagged as ``action permanent failure'' (what may trigger backup processing) +\item the action is actually suspended for $C_i$ seconds +\end{itemize} +If then a new message is sent to the action, and $C_i$ seconds have not yet elapsed, the action is flagged as having failed without being re-tried again\footnote{During the analysis for this paper, it was seen that actually $C_r$ retries are attempted in v3, but each of them will never actually re-try the action. This is a software bug, which does not cause any harm and thus will not be fixed in v3. The new implementation in v4 will obviously not inherit this problem}. This is done in an effort to reduce resource utilization and prevent the system from slowing down e.g. by too-many retries to a remote server that went offline. + +With transactional output mode in \emph{rsyslog v4}, the logic above can no longer work. First of all, retrying single actions does not help, because all of the current transaction needs to be resubmitted. As such, the upper layers need to be notified of failure. Then, they need to resubmit the batch. In that design, the lower layer needs to return immediately after detecting the failure. Recovery handling is now to be done when the next transaction is started. However, we must make sure that we do not do excessive retries. So retry processing is only to be carried out if it was not tried less than $C_i$ seconds ago. + +The required functionality can be implemeted by a \emph{prepareAction} function that readies the action for processing if there is need to do so. That function is then called in all entry points before anything else is done. Then, actual processing is carried out and the resulting action state be used to generate the return code for the upper-layer caller. Find below a rough pseudocode to do so: + +\lstset{language=python} +\begin{lstlisting} +def prepareAction(): + if state == rtry: + try recovery (adjust state accordingly) + if state == rdy: + beginTransaction() [output plugin] + +def processMessage(message): + prepareAction() + if state == itx + doAction(message) [output plugin] + return getReturnCode() + +def doEndTransaction(): + prepareAction() + if state == itx + endTransaction(); [output plugin] + return getReturnCode() +\end{lstlisting} + +\subsection{Output Subsystem Layers} +The rsyslog engine is organized in layers, where each layer is represented by the dominating object: + +\begin{figure} +\includegraphics[scale=0.75]{rsyslog_output_layers.jpeg} +\label{rsyslog output layers} +\end{figure} + +If looking at the data flow, a queue dequeues batches of messages, which are than run through a generic action system and put into output plugins. Note that on the batch layer, only batches are supported as units of work, whereas the action layer is message-oriented but supports transactions of multiple messages. This is done by indicating when a transaction necessarily needs to end (that point being the end of batch from the batch layer). + +The plugins can be written by third parties and are roughly comparable to minidrivers. The generic action system provides all complexity of action processing wheras the output plugin provides a limited set of callbacks that enable the generic framework to talk to the actual destination system. As such, writing outputs is a very simple task. However, rsyslog does not limit the creation of very complex outputs, which may be able to offer superior performance for some destinations. + +\subsection{Output Failure} +\subsubsection{Cases} +When an output action is called, it may encounter a failure condition. In general, there are two different cases: +\begin{enumerate} +\item action caused failures +\item message-content caused failures +\end{enumerate}. + +Failures rooted in the action are things like broken network connections, file systems run out of space or database servers that are down. Most importantly, the failure is not related to message content. As such, it is appropriate to retry the action with the same message until it finally succeeds (assuming that someone restores the system in question to proper operation). We can not expect that the problem is cleared just by discarding the current message and re-trying with the next one. + +In my view, action caused failures are the far majority of all failures. For rsyslog versions 3 and below, all rsyslog-provided plugins consider failures to be action-caused and thus potentially recoverable by simple retry. With the only exception being fatal error conditions that render the whole action unusable. + +David Lang pointed out, that there may also exist error conditions that are not caused by the action (or the subsystem it talks to) itself, but rather by message data. He provided the following samples where message content can cause permanent issues with action execution: + +\begin{itemize} +\item unicode text causing grief +\item dynafile hits a read-only file +\item basicly data-driven things that trigger bugs in the message delivery +mechanism in some form. +\end{itemize} + +As David Lang said ``In an ideal world these would never happen, but for most output types I can think of some form of corrupt input that could cause that message to fail.''. +So this class of failure conditions actually exists. No matter how often the action retry mechanism is called, it will never succeeds (one may argue that the read-only dynafile is fixable, but we could replace that sample with an invalidly generated filename). The proper cure for these actions is to find the offending one and discard it. + +In conclusion, actions need to return different error states for these two different types of failures. Traditionally, RS\_RET\_SUSPENDED is returned when an action specific failure is hit. Most existing plugins also do this if a message-related failure occured, simply because they did not yet know that this situation exists. However, plugins also return different error codes, and at least these can be treated to mean message-permanent failures. To support this, a change to plugins is still required, because many simple return SUSPENDED state if anything went wrong (replacing the real error condition with SUSPENDED). A dedicated PROBABLE\_INVALID\_MSG return state is probably useful so that an output plugin can convey back that it consideres the message to be bad. On the other hand, this implies that the plugin must try to detect those, what means that the developer must think about all potential message-causes problems. That approach can be considered unreliable and as such it may be better not to provide such a dedicted state. + +\subsubsection{Handling of Failures} +In spite of the two different failure cases, different handling is needed for them. The action-based failure cases can and must be handled on the action level. As transactions abort when a failure occurs, support from the upper ``batch layer'' is necessary in order to handle resending batches of messages. + +For message-caused failure cases, the offending message must be found and then be discarded. A complexity here is that while a failure-causing message is being searched for, an action-based failure might occur. In that case, first the action-based failure condition must be solved, before the search for the problem message can continue. + +One approach might be that when the action-layer conveys back an action-caused failure (SUSPENDED), the batch layer knows that it simply needs to restart the full transaction (but not start an ``invalid message search''). If a message-based error condition is conveyed back, the batch system can not restart the full batch. Instead, it needs to enter search mode, where it creates partitions of the original batch, and calls itself recursively (at least in theory) on each of the subsets. + +Then, the same handling applies until either a failing message has been found or all messages have been successfully processed. Note that in the recursive step, action-based failures are recovered by full batch resubmits. This solves the above-mentioned complexity in a consistent way. + +If a binary-search-like method is used to detect failing records\footnote{This was originally suggested by David Lang.}, recursion may not really be an issue, as the recursion depth is limited to $\log_2 |B|$ where $B$ is the message batch. + +A message-caused failure can be rooted in one or more messages. One important question is if it is expected that the failure is caused by a single or multiple messages. Both is possible, so it is a question of probability. If we assume that it is more probable that a single messages causes the problems, it is useful to immediately return back to full batch submission of transactions once a problem-causing message has been identified. But then, if there are multiple problem-causing messages inside the batch, we may need many more iterations. + +If, on the other hand, we assume that it is more probable that multiple messages cause problems, it may make sense to keep resubmitting only subsets of the batch. However, then the performance is suboptimal if actually only one message was problematic. A solution might be to pick a compromise, e.g. first assume that a single message is problematic, but assume the opposite as soon as a second message with problems has been found. + +A potential algorithm for processing $n \le |B|$ messages from batch $B$ is described below. In the pseudocode, a ``processable'' message is one that neither is already committed nor had a permanent failure with this action. The term ``mpf'' means ``message permanent failure'' for this action (this will later be described in a batch state set). + +\begin{small} +\lstset{language=python} +\begin{lstlisting} +def submitBatch(B, n): + foreach processable message in + (first [at most] n messages of batch): + call processMessage + if action-caused failure: + retry full batch + if action-caused permanent failure: + mark all n messages as mpf + return + if auto-commit: + mark commited messages in batch as committed + if message-caused failure: + if n == 1: + mark message as mpf + return + else: + call submitBatch(B, n/2) + call submitBatch(B, n/2) +\end{lstlisting} +\end{small} + +After submitBatch() has completed, all messages are either committed or in mpf state. + +Note that an action-caused permanent failure occurs if an action-caused failure can not be resolved with the operator-configured number of retries. It will never occur if the user configured infinite retries. While an action is suspended, all calls will result in an action-caused permanent failure. Please keep in mind that these will be resubmitted to any backup actions inside the action unit, so the action's ability to cause permanent failure states is vital for a number of use cases (backup syslog server, to name just one). + +Batch processing inside an action unit thus can follow these strucuture: + +\begin{algorithm} +\caption{processBatch(B)} +\begin{algorithmic} +\FORALL{action $a$ in action unit} + \IF{execute action only on messages that failed before} + \STATE $n = |\text{messages in batch in mpf state}|$ + \STATE change mpf state back to ready + \ELSE + \STATE $n = |B \setminus \text{msgs with state discard}|$ + \STATE change all message states $\ne$ discard to ready + \ENDIF + \IF{$n >0$ } + \STATE call submitBatch(B, n) for action $a$ + \ENDIF +\ENDFOR +\end{algorithmic} +\end{algorithm} + +\paragraph{Why is it Important to differentiate the failure cases?} +This text originates from the mailing list and must be merged in. I provide it in the form it is, so it will not be forgotten (plus, it conveys the information). + +One may think that it is not necessary to differentiate between action-caused and message-caused failures. However, not doing so introduces subtle issues, because +then you either + +A) do not need the batch logic at all (because the action is configured for +infinite retries) + +Or + +B) you loose many messages if the action is not configured for infinite +retries and you have a longer-duration outage e.g. on a database server. +Let's say it is offline for a couple of hours, then you lose almost +everything in that period + +To prevent this, you need two different retry methods. + +One may argue that it is hard to differentiate between the two failure cases. This is correct. Buit I think it mostly depends on the quality of the output module. + +First of all, ``mostly'' implies that there may be some other cases, where it +really is impossible to differentiate between the two. In that case, I would +treat the issue as an action-caused failure. There are two reasons for this: + +1) rsyslog v3 currently does this always and not even a single person +complained about that so far. This is an empiric argument, and it does not +mean it caused problems. But it carries the co-notation that this seems not +to be too bad. + +2) If we would treat it as message-caused failure, we would no longer be able +to handle extended outages of destination systems, which I consider a vitally +important feature. + +When weighing the two, I know of lots of people who rely on 2), in sharp +contrast to knowig noone having problems with 1). So my conclusion is that it is +less problematic to define an otherwise undefinable failure reason to be +action-caused. Even more so as I assume this problem only exists in the +minority of cases. + +Now back to the quality of the output module: thinking about databases, their +API is usually very good at conveying back if there was a SQL error or a +connection abort. So while a SQL error may also be an indication of a +configuration problem, I would strongly tend to treat it is a being +message-caused. This is under the assumption that any reasonable responsive +admin will hopefully test his configuration at least once before turning it +into production. And config SQL errors should manifest immediately, so I +expect these to be fixed before a configuration runs in production. So it is +the duty of the output module to interpret the return code it received from +the API call and decide whether the failure is more likely action-caused or +message-caused. For database outputs, I would assume that it is always easy +to classify failures that must be action-caused, especially in the +dominating cases of failed network connections or failed servers. + +For other outputs it may not be as easy. But, for example, all stream network +outputs can detect a broken connection, so this also is a sure fit. + +For dynafiles, it really depends on how hard the output module is tries to differentiate +between the two failure cases. But I think you can go great length here, too. +Especially if you do not only look at the create() return code, but, iff a +failure occurs, you do more API calls to find out the cause. + +So I think the remaining problem is small enough to cause not too much issues +(and if so, they are unavoidable in any case). In conclusion, the two failure states are not only necessary, but can sufficiently sure enough be detected. + +\subsection{Random Topics} +I have begun to gather material from the mailing list in this section, because I feel it may be useful for others as well. Right now, the information is well hidden in the mailing list archives and there may be value in combining it all in one place. + +Due to the nature of this material, there is no specific organization between the subchapters and also formatting and language doesn't deny its rooting in the mailing list. + +\subsection{Reliability of Message Dequeueing} +A batch is actually dequeued when it is taken off a queue. So if at that point we +have a system power failure (for whatever reason), the messages are lost. +While the rsyslog engine intends to be very reliable, it is not a complete +transactional system. A slight risk remains. For this, you need to understand +what happens when the batch is processed. I assume that we have no sudden, +untrappable process termination. Then, if a batch cannot be processed, it is +returned back to the top of queue. This is not yet implemented, but is how +single messages (which you can think of an abstraction of a batch in the +current code) are handled. If, for example, the engine shuts down, but an +action takes longer than the configured shutdown timeout, the action is +cancelled and the queue engine reclaims the unprocessed messages. They go +into a special area inside the .qi file and are placed on top of the queue +once the engine restarts. + +The only case where this not work is sudden process termination. I see two +cases: + +a) a fatal software bug +We cannot really address this. Even if the messages were remaining in the +queue until finally processed, a software bug (maybe an invalid pointer) may +affect the queue structures at large, possibly even at the risk of total loss +of all data inside that queue. So this is an inevitable risk. + +b) sudden power fail +... which can and should be mitigated at another level + +One may argue that there also is + +c) admin error +e.g, kill -9 rsyslogd +Here a fully transactional queue will probably help. + +However, I do not think that the risk involved justifies a far more complex +fully transactional implementation of the queue object. Some risk always +remains (what in the disaster case, even with a fully transactional queue?). + +And it is so complex to let the messages stay in queue because it is complex +to work with such messages and disk queues. It would also cost a lot of +performance, especially when done reliably (need to sync). We would then need +to touch each element at least four times, twice as much as currently. Also, +the hybrid disk/memory queues become very, very complex. There are more +complexities around this, I just wanted to tell the most obvious. + +So, all in all, the idea is that messages are dequeued, processed and put +back to the queue (think: ungetc()) when something goes wrong. Reasonable +(but not more) effort is made to prevent message loss while the messages are +in unprocessed state outside of the queue. + +\paragraph{More reliable can actually be less reliable} +On the rsyslog mailing list, we had a discussion about how reliable rsyslog should be. It circles about a small potential window of message loss in the case of sudden fatal failure. Rsyslog can be configured to put all messages into a disk queue (instead of main memory), so these messages survive such a powerfail condition. However, messages dequeued and scheduled for processing during the power outage may be lost. + +I now consider a case where we have bursty UDP traffic and rsyslog is configured to use a disk-only queue (which obviously is much slower than an in-memory queue). Looking at processing speeds, the max burst rate is limited by using an ultra-reliable queue. To avoid using UDP messages, a second instance could be run that uses an in-memory queue and forwards received messages to the one in ultra-reliable mode (that is with the disk-only queue). So that second instance queues in memory until the (slower) reliable rsyslogd can now accept the message and put it into the reliable queue. Let's say that you have a burst of $r$ messages and that from these burst only $r/2$ can be enqueued (because the ultra reliable queue is so slow). So you lose $r/2$ messages. + +Now consider the case that you run rsyslog with just a reliable queue, one that is kept in memory but not able to cover the power failure scenario. Obviously, all messages in that queue are lost when power fails (or almost all to be precise). However, that system has a much broader bandwidth. So with it, there would never have been r messages inside the queue, because that system has a much higher sustained message rate (and thus the burst causes much less of trouble). Let's say the system is just twice as fast in this setup (I guess it usually would be *much* faster). Than, it would be able to process all r records. + +In that scenario, the ultra-reliable system loses $r/2$ messages, whereas the somewhat more "unreliable" system loses none - by virtue of being able to process messages as they arrive. + +Now extend that picture to messages residing inside the OS buffers or even those that are still queued in their sources because a stream transport blocked sending them. + +I know that each detail of this picture can be argued at length about. + +However, my opinion is that there is no "ultra-reliable" system in life, only various probabilities in losing messages. These probabilities often depend on each other, what makes calculating them very hard to impossible. Still, the probability of message loss in the system at large is just the product of the probabilities in each of its components. And reliability is just the inverse of that probability. + +This is where *I* conclude that it can make sense to permit a system to lose some messages under certain circumstances, if that influences the overall probability calculation towards the desired end result. In that sense, I tend to think that a fast, memory-queuing rsyslogd instance can be much more reliable compared to one that is configured as being ultra-reliable, where the rest of the system at large is badly influenced by this (the scenario above). + +However, I also know that for regulatory requirements, you often seem to need to prove that a system may not lose messages once it has received them, even at the cost of an overall increased probability of message loss. + +My view of reliability is much the same as my view of security: there is no such thing as "being totally secure", you can just reduce the probability that something bad happens. The worst thing in security is someone who thinks he is "totally secure" and as such is no longer actively looking at potential issues. + +The same I see for reliability. There is no thing like "being totally reliable" and it is a really bad idea to think you could ever be. Knowing this, one may begin to think about how to decrease the overall probability of message loss AND think about what rate is acceptable (and what to do with these cases, e.g. "how can they hurt"). + +\paragraph{Different Use Cases} +As David Lang pointed out, there exist different use cases for different levels of reliability. Most importantly, there exist use cases that do not demand very high throughput but rather ultra-realiability of the queue system. Here, ultra-reliability is just another word for the queue being of ``audit-grade''. Even if the queue provides audit-grade, the overall system is only then of audit-grade when all other components - most notably the transport protocols spoken by the inputs and outputs - are also of audit-grade. Most importantly, this means that an audit-grade system purely based on the IETF syslog protocol series can not be build. + +Used together with truly reliable protocols \emph{and} senders that block processing until a final acknowledgement has been received, an audit-grade system can potentially build based on rsyslog. To do so, an audit-grade queue subsystem is required, which is not present in releases less than 4.1.? (most importantly, v2 and v3 do not provide this capability). + +\subsection{Audit-Grade Queue Operations} +\subsubsection{Perquisites} +Audit-grade queue operations certain perquisites: +\begin{itemize} +\item rsyslog engine is of version 4.1.? or greater +\item disk-only queue type +\item checkpoint interval set to 1 +\item queue is configured to not permit losing any messages\footnote{The queue has several settings that can be used to fine-tune situations in which it may discard messages intentionally. All of these must be turned off. Most importantly, that means the producer is blocked for an infinite time if the queue is full.} +\item queue consumer must also be of audit-grade +\end{itemize} +Only when these prequisites are met, queue operation can be considered of being audit-grade. Note that when message loss in case of sudden fatal failure and similar incidents is acceptable, neither disk-only queues nore a checkpoint interval of 1 is necessary. Such a configuration can also be build with rsyslog v3, which is up to that level. + +Note that in the sections below we describe the implementation in broader terms. Most importantly, we do not restrict ourselves to disk-only queue storage drivers. This is important, because it simplifies design and opens the capability to introduce new, possibly faster-performing, queue storage drivers in the future. + +But it is important to keep in mind that a concrete queue is only of audit-grade if it matches all the perquisites given here, most importantly with the right configuration. + +\subsubsection{Implementation Alternatives} +Messages, or more precisely objects\footnote{While rsyslog deals with messages, the queue is designed to handle any type of thing that is represented as an rsyslog object. This is considered useful as queues may at some time contain other things than just messages, so we keep it generic.}, are enqueued by the queue producer (either an input module or the main message queue's consumer). The enqueue operation is completed only when the message has been successfully accepted by the queue storage driver. Then and only then the producer is permitted to remove the object from its own storage system. A rough sketch is given in algorithm \ref{alg_q_enq}. + +\begin{algorithm} +\caption{enqueueObject($o$)} +\begin{algorithmic} +\label{alg_q_enq} +\STATE lock queue mutex +\WHILE{queue is not ready for enqueue} + \STATE wait on queue to become ready +\ENDWHILE +\STATE call queue store driver to add $o$ +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +The dequeue-operation is more complex. We must ensure that each object stays in the queue until it is finally processed. Hereby, an object is finally processed, when processing of it has been completed. Remember that to enhance performance, objects are dequeued in batches of many. So at any given time, multiple messages may be processed, but not necessarily have finally completed doing so. If another worker thread then tries to obtain a new batch for processing, those ``in-process'' message must not be handed out a second time. Also, if a sudden fatal failure occurs during processing, queue operation must restart at the point of last commit. This means that all ``in-process'' messages need to be changed back to ``no processed'' state and be restarted again. In those cases the (acceptable) slight message duplication can occur. + +In our design, we differentiate between ``logical'' and ``physical'' dequeuing of batches. If a batch is generated for processing, it is logically dequeued --- in the sense that no other batch generating request will be able to receive another copy of these messages. If no exceptional situation happens, those messages will be processed and thus can be considered consumed under normal circumstances. + +However, actual deletion from the physical queue storage happens only after the batch is fully processed. At this point, all objects have been acknowledged by their destinations, which now have the responsibility for the object's survival. Consequently, we can delete them from the queue store. This process is considered the ``physical'' dequeue of the object. + +In order to find some simpler terms, we will call the logical dequeue operation just ``dequeue'' and the physical dequeue operation ``delete''. This is consistent with all previous work on rsyslog and thus probably leads to the least surprise when reading older source code and documentation. + +A first idea for a deletion is given in algorithm \ref{alg_pdeq_batch_1} (remember that $O(b)$ contains all objects within the given batch $b$, this is \emph{not} $O$-notation and should probably in the future be replaced by something else). + +\begin{algorithm} +\caption{deleteBatch($b$), first approach} +\begin{algorithmic} +\label{alg_pdeq_batch_1} +\STATE lock queue mutex +\FORALL{$o \in O(b)$} + \STATE find $o$ in queue storage + \STATE remove $o$ and keep queue structures intact +\ENDFOR +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +This algorithm is simple, but requires searching the queue store for the object to be deleted -- a potentially lengthy operation. However, we can improve the searching process if we know more about the inner structure of batch objects. It seems appropriate to dequeue objects in queue-sequential order. A drawback of doing so is that we must prevent other worker threads from trying to dequeue concurrently. This is not really a drawback. We need to guard dequeue operations by a mutex in any case, because otherwise internal structures can not be kept consistent. Practical experience and testing have shown that many small dequeue operations cause a lot of locking contention and as such badly affect performance. So it actually is a welcome enhancement to aquire the queue lock only once for the whole batch dequeue operation. As dequeing is a comperatively fast operation, the lock is not held for extended periods of time. + +A first approach to this functionality is shown in algorithm \ref{alg_ldeq_batch_1}. Note that $C_{mBatch}$ is the configured maximum number of elements inside a batch, $i$ is an index to address the objects inside the batch. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers.jpeg} +\end{center} +\caption{\textbf{Queue Store Pointers}: boxes represent queue entries, colored boxes entries with objects. Objects in green are unprocessed, in blue are dequeued but not deleted and those in gray have already been deleted. White indicates not yet used entries. Gray objects may be overwritten at any time. Their entries are actually free, we have used the gray color primarily to indicate there once existed objects. Each queue pointer points to the next entry to process.} +\label{fig_queue_ptr} +\end{figure} + +\begin{algorithm} +\caption{dequeueBatch($b$)} +\begin{algorithmic} +\label{alg_ldeq_batch_1} +\STATE lock queue mutex +\STATE $0 \to i$ +\WHILE{queue non-empty and $i < C_{mBatch}$} + \STATE obtain next obj $o$ from queue store + \STATE advance logical dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +A key concept is somewhat hidden in \marginpar{queue pointers} \emph{advance logical dequeue position}. Each queue store is purely sequential, with objects being enqueued at one ``end'' of the store and dequeued at the other. Of course, each queue store has only finite capacity, but we ignore this to explain the overall picture. A queue can be implemented by two pointers: one that points to the tail of the queue, where new messages are enqueued and one that points to the head of it, where new messages are dequeued. The idea is now to duplicate the dequeue pointer and split it into one for (logical) dequeue and one for deletion. Figure \ref{fig_queue_ptr} shows this three-pointer approach. Now, we can simple advance either the dequeue or deletion pointer, depending on operation, and do not need to find the first dequeue position inside the queue store. The dequeue pointer always points at it. This mode can be implemented with all currently existing queue storage drivers (but the sequential disk driver may need to use a second file handle or stream object instead of two pointers). + +This makes an efficient implementation of algorithm \ref{alg_ldeq_batch_1} possible: when it logically dequeues, it just needs to advance the dequeue pointer. So the algorithm executes in $O(n)$ time where $n$ specifies the number of elements to dequeue with an upper bound of $C_{mBatch}$. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers2.jpeg} +\end{center} +\caption{\textbf{Physically Dequeueing Messages}: In this sample, we have two batches. With multiple workers, they may be deleted in any order.} +\label{fig_queue_ptr_deq} +\end{figure} + +Furthermore, we can also improve algorithm \ref{alg_pdeq_batch_1}: Consider that each batch is logically dequeued as an atomic operation. That means all batch objects form a sequential subset of the queue. Figure \ref{fig_queue_ptr_deq} shows the situation when two batches have been dequeued. So the costly ``find'' operation now needs to be carried out only once at the beginning of the batch. As all other objects are sequential, once we have found the batch begin inside the queue, we can simply delete the $|b|$ elements in queue-sequential order after it. So the cost of the find operation can be reduced from $O(|b|)$ to $O(1)$. + +We can even reduce the remaining cost of the find operation. If the batch to be deleted is right at the queue's head (as is ``B1'' in the figure), the ``find'' immediately terminates with the first element and incurs no cost at all. The situation is different if the batch is not at the queue head, ``B2'' is an example for that (assuming that ``B1'' has not yet been dequeued). We would now still need to search over the objects that are not part of the batch and can then finally get to the object at the head of the batch in question. For queue storage drivers that support random access to queue elements, storing a simple pointer to the batches' queue head element further improves the situation and enables $O(1)$ access to the queue element. This is indicated by the dotted lines in figure \ref{fig_queue_ptr_deq}. Once the head of the queue has been found, two things can happen (depending on the capabilities of the queue storage driver): + +\begin{enumerate} +\item the head element can be flagged as ``this and next $n$ elements are deleted'' +\item all elements are actually deleted +\end{enumerate} + +Note that a mixed form is also possible (and probably useful for our \emph{singly} linked list storage driver: there, some $n'$ elements be actually deleted and the head element is flagged as ``this and next $n - n'$ elements are deleted''. Note that in the linked-list case, all but the first elements can be deleted with ease\footnote{It can be considered to change from a singly-linked list to a doubly-linked list, if the benefit outweighs the extra effort required.}, so probably just the head would stay inside the queue. Note that removing elements off the queue, where possible, is useful because it frees resources. On a busy system, freeing messages as soon as possible can prevent message loss (in non-audit-grade setup) or system slowdown. So it should be done when possible. + +If we have a purely sequential queue storage driver (currently the sequential disk driver), finding and updating the head element is not an option. Even in this case, we can observe that the batch at the actual deletion pointer will eventually be submitted for deletion. So a route to take is to create a list of elements that can be deleted as soon as the physical dequeue pointer reaches any of these elements. We call this the \marginpar{to-delete list}``to-delete list''. To facilitate processing, this list must be ordered in sequence of dequeing. This information may not be available from the storage subsystem itself, but it can easily be generated. To do so, a strictly monotonically increasing counter is kept with each logical dequeue operation and stored as part of the batch\footnote{As this must be done via the usual computer-implemented modular arithmetic, we must be careful that we do not see repetion of values because of overflows. Each day has $60 \cdot 60 \cot 24 = 86,400$ seconds (ignoring the subleties of UTC). Now let's assume that we have a moderately-busy system with 1,000 messages per second. We further assume, to be on the save side, that each message is processed inside its own batch. So we have $86,400,000$ batches per day. If we now use a typical $32$-bit integer for generating the batch IDs, we the unique range will be used up after +$$\frac{2^{32}}{8640000} \approx 497 \text{ days}$$ +days of uninterrupted rsyslog operation. While this sounds somewhat save, it goes down to approximately 10 days of messages are submitted at rate of 50,000 messages per second (which is high, but not unheared of). So it is strongly advised to use 64 bits, which we consider to be save, because for our 1,000 messages per second the range would be exhausted only after +$$\frac{2^{64}}{8640000} \approx 2.135 \cdot 10^{11} \text{ days}$$ +which equals approximately $584,500,000$ \emph{years}. So even at a rate of one million messages per second, the range would be sufficient for over 500,000 years of continuos operations -- that should be far sufficient.} +An example: let us assume that ``B2'' was submitted for deletion first. Then, the head of ``B2'' is not at the queue's delete pointer. As such, no action can be carried out immediately. So the batch head pointer is stored into a ``to be deleted'' list. Processing continues. Some time later, batch ``B1'' is submitted for deletion. Now, the head pointer is at the head of the delete list, as such all batch elements are dequeued. Then, the ``to be deleted'' list is checked, and ``B2'' is found in it. Now, ``B2'' is at the head of the (new) deletion pointer and can also be removed. So, ultimately, all messages are physically dequeued. This is more formally describe in algorithm \ref{alg_phys_deq_seq_store}. In that pseudocode, we made a simplification by always putting the to be deleted batch in the ``to-delete'' list, which then enables us to use somewhat more generic code to carry out the work. + +Note that there is a price to pay for deletions via the ``to-delete'' list: if a sudden fatal failure happens during processing, the set of duplicate messages is increased. For example, if a fatal failure happens after ``B2'' has been fully processed and scheduled for deletion, but \emph{before ``B1'' is also submitted for deletion}, ``B2'' will be reprocessed after recovery. This would not happen if ``B2'' would have been removed from the queue. + +\begin{algorithm} +\caption{deleteBatch($b$)} +\begin{algorithmic} +\label{alg_phys_deq_seq_store} +\REQUIRE queue mutex is locked by caller +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.deletePtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.deletePtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\end{algorithmic} +\end{algorithm} + +\paragraph{Warp-Up of Queue Delete Operations} +When evaluating which route to take, the ``to-delete'' list approach looks elegant for all cases. The negative side effect of potentially increased message duplication currently does not even exist: today, the sequential disk queue storage driver permits only a single worker thread and thus there always will be only one thread at a time. Even if we remove that limitation, message duplication could not be avoided, as stated in the algorithm description above. What remains are the other queue storage drivers. However, they operate in-memory, so message duplication will not happen simply because all messages will be lost on sudden fatal failure. The advantage of limited message duplication only exists in the so-far hypothetical case of a random-access, audit-grade disk queue storage driver. Thus, the decision could be postponed unless that happens (if it ever does). + +From a code complexity point of view, the ``to-delete'' list approch is definitely advantagous. Not only because of the reduced number of algorithms required. We also do not need to maintain unique batch IDs and all the logic associated with them. + +The other aspect to look at is memory consumption. Assuming that we delete the actual objects, just not their containers inside the queue, extra memory consumption is not really that worse. More importantly, currently only the linked-list queue storage driver can benefit at all, because it is the only driver capable of deleting queue entries in mid-queue. All others, including the array memory driver, do not have this capability. + +From a performance point of view, the ``to delete'' list approach looks approximately as good as the others, with some mild better performance for some storage drivers for a non-``to delete'' list approach. This can be mitigated, especially if the potentially somewhat-costly maintenance of the ``to-delete'' list is slightly optimized and the algorithm actually checks if the to be deleted batch is right at the queue's delete pointer position. The improved code simplicity, together with current CPU's code caching, may even result in an otherwise not expected speedup. + +In conclusion, we will implement the ``to-delete'' list approach on the queue layer (above the queue storage drivers). However, we will leave the window open to permit overwriting it with queue storage driver specific functionality. How to do this will not be specified now, as there is currently no need and we do not even know if there ever will be. However, we retain the discussion on the various modes as well as the relevant algorithmic discussions and data structurs inside this paper so that it is readily available should need arise. We also think this is important so that everybody later knows that the decision was made based on good argument and not by accident (we consider this useful in another design enhancement attempt). + +\paragraph{Processing Sequence} Looking at the processing sequence, we notice that always objects are dequeued, then processed and then deleted. Then, the whole process starts again. In particular, this meanss that after the previous batch has been deleted, the next batch will be dequeued. Now consider that we need to have exclusive access to the queue for both of these operations. As such it seems natural to combine this into a single step, further reducing potential locking contention. + +Note that a side-effect of this approach is that messages can be deleted only when a new batch is dequeued. With current design, this means that at least one message must reside inside the queue. Otherwise, the last batch will not be deleted. However, this something that can (and must!) be solved on the queue worker layer, in that it deletes a batch when the queue is empty. + +This leads us to the implementation of dequeueBatch() and deleteBatch() shown in algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. Note that $l$ is a flag variable that indicates if the queue is already locked. + +\begin{algorithm} +\caption{dequeueBatch($b$): final version} +\begin{algorithmic} +\label{alg_deq_batch_final} +\STATE lock queue mutex +\STATE call deleteBatch(b, 1) +\STATE $0 \to i$ +\WHILE{queue non-empty and $i < C_{mBatch}$} + \STATE obtain next obj $o$ from queue store + \STATE advance dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + + +\begin{algorithm} +\caption{deleteBatch($b, l$): final version} +\begin{algorithmic} +\label{alg_del_batch_final} +\IF{queue not yet locked (test via $l$)} + \STATE lock queue mutex +\ENDIF +\FORALL{objects $o$ in $b$} + \STATE destruct $o$ +\ENDFOR +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.deletePtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.deletePtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\IF{queue not yet locked (test via $l$)} + \STATE unlock queue mutex +\ENDIF +\end{algorithmic} +\end{algorithm} + +\subsubsection{Queue Stores} +Currently, rsyslog supports three different types of queue store drivers: + +\begin{itemize} +\item memory array +\item memory linked list +\item disk sequential file +\end{itemize} + +They all provide an abstracted sequential queue store as shown in figure \ref{fig_queue_ptr} on page \pageref{fig_queue_ptr}. + +Obviously, some differences exist. Most importantly, the disk sequential file driver does \emph{not} support more than one queue worker thread (in order to prevent excessive disk activity and the subtle issues with rewriting parts of sequential files). So if this driver is used, the queue automatically limits itself to a maximum of one worker thread (even if user configuration settings + +Different queue store drivers have different properties: + +\begin{tabular}{|l||l|l|l|}\hline + & array & linked list & seqential file \\ \hline +pointer type & integer index & memory address & file number and \\ + & & & offset within file \\ \hline +physical access & random & random & sequential \\ \hline +remove middle & no & yes & no \\ +elements & & & \\ \hline +access to $n$-th& $O(1)$, index:& $O(n)$, follow & not supported \\ +element & $n \mod C_{mMsg}$ & pointer links & \\ \hline +speed & fastest & fast & slow \\\hline +mem overhead & large & some & almost none \\\hline +reliability & reliable & reliable & audit-grade\footnote{if configured correctly}\\ +\hline +\end{tabular} + +\subsubsection{Implementation} +The actual implementation will be based on algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. The rsyslog v3 queue storage driver will be extended one additional method, which permits non-destructive dequeueing of elements. As such, the driver now has the $qAdd()$, $qDeq()$, and $qDel()$ entry points (together with the usual construction and destruction entry points). The queue drivers must support the three pointers for enqueue, dequeue and delete. The ``to-delete'' list will be maintained on the upper queue layer (and not the queue driver layer). This functionality will be optimized so that if a batch to delete is right at the queue's delete pointer, it will immediatly be deleted and not be sent to the ``to-delete'' list. This is especially important with the sequential disk driver, as the condition here always is true (and thus the driver can pretend this in the relevant API without even comparing any pointers -- what would otherwise quite complicated in this driver. + +The full list of the queue store driver interface is: + +\paragraph{qConstruct} Initializes the queue store. + +\paragraph{qDestruct} Destructs the queue store, including all messages that may still be present in it. + +\paragraph{qAdd} Enqueue a new object into the queue. Note that this entry point must only be called when the queue is non-full. + +\paragraph{qDeq} Non-destructive dequeue of the object at queue head. Dequeue pointer is advanced. + +\paragraph{qDel} Delete the object at queue head. Delete pointer is advanced. + +Disk queue store drivers may support additional internal functions. However, they should not be exposed to the rest of the queue subsystem. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{queue_msg_state.jpeg} +\end{center} +\caption{Logical Message States during Queue Processing} +\label{fig_queue_msg_state} +\end{figure} + +Figure \ref{fig_queue_msg_state} shows a logical message state diagram during queue processing. There is no actual state variable, but rather the processing flow demands these state. Note that the state transition from ``dequeued'' to ``queued'' only happens after a fatal failure and a successful system recovery. So this is a rather exceptional case. + +Another subtle issue is that we now need two different queue size counters: one for seeing when the queue is physically full and one for detecting when there are no more messages to be dequeued. + +As a simplification, support for ungetting objects can be removed (as objects never leave the queue), what also means that cancel-processing is probably less complex. + +\paragraph{Sequential Disk Queue Store Driver} +The enequeue, deqeueue and delete pointers must be implemented via three stream objects. Most importantly, the dequeue stream must be configured not to delete files when it closes them. A side-effect of this implementation is that data is actually read twice, once to actually obtain it and a second time to delete it. This could only be avoided by an overall redesign on how the disk queue works. + +\subsubsection{Checkmarks} +The following things need to be verified in the actual implementation. + +\paragraph{Queue Full} +Is it possible to set an infinte timeout on queue full condition during enqueue? If not, we must provide it. + +\paragraph{Termination the Queue} +If we cancel a worker, we need to start from the physical dequeue pointer and pull everything that is not scheduled for deletion - NOT from the logical dequeue pointer. + +\paragraph{Failed Messages} +If a message fails on a detached action queue, no backup processing is available (because we detect the failure at a point where the message is already considered processed from the main queue's point of view. We need address this and have two options: + + +I see two approaches at handling this: + +a) we enable an action to configure a backup file that shall receive all +message permanent failures. This is simple (not only to implement but to +configure and understand) + +b) we push the failed message back to the main queue, but with an indication +that it failed in an action. This is harder to implement and most importantly +harder to understand/configure, but more flexible + +\section{Network Stream Subsystem} +The idea of network streams was introduced when we implemented RFC5425 (syslog over TLS) in 2008. The core idea is to encapsulate all stream-oriented network data transfer into a single transport layer and make the upper layers independent of actual transport being used. This is in line with the traditional layer approaches in communication systems. + +Under this system, the upper layer provides plugins to send and receive streams of syslog data. Framing is provided by the upper layer. The upper layer itself is integrated in input and output plugins, which then are used to provide application-level syslog message objects to and from the rsyslog core. To these upper layers, the netstream layer provides reliable and sequenced message delivery with much of the same semantics as a usual TCP stream. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{tls.jpeg} +\end{center} +\caption{Objects at the Network Stream Layer} +\label{fig_netstream_objects} +\end{figure} + +At the netstream layer, we have a small set of generic classes, which are used for setup of the drivers and driver parameters. This is a very thin layer, mostly a wrapper. Once an actual lower-level netstream driver has been loaded, all parameters are passed through to it. + +Please note that both in theory and practice netstream drivers may call back into different netstream drivers. For example, the GnuTLS RFC5425 driver loads and calls back into the plain tcp driver, simply because that driver provides part of the required functionality and there is no point in re-implementing it for GnuTLS. + +The netstream driver layer does not only provide read and write calls but supports i/o multiplexing. To do so, it offers an interface that follows select() semantics. That permits an upper-layer comonent to request being blocked unless some data arrives. Note that due to the subleties in TLS processing, the upper layer may be awoken while there is no upper-layer work to do. This will properly be indicated by the netstream subsystem, is not an error and must be accepted and poperly handled by the upper layer. + +Using the nestream layer, we do not need to modify the input and output plugins while at the same time we can add additional transport providers. One weak spot in this design is the current configuration process. With the current system, we need to provide one configuration statement per driver property and we need to hardcode this. So if a new driver would require new properties, we still would need to modify the upper layers. This is unfortunate, but the current config system does not provide for any better way to handle the situation. Once we are able to create a new config system, we will address this by providing the ability to pass a string of parameters onto the driver, which will then have the ability to parse its content. So once we do this, we need to modify the driver interface, but the end result would be a simlification. + +So far, only drivers for GnuTLS and plain tcp are provided. However, during the design of the layer we also looked at openssl and Mozilla Network Security Services as well as kept an eye on the needs of Kerberos. In theory, it should not be a major problem to write drivers for these systems (but it most probably still is a lot of work to do). + +A final note on Kerberos: in order to keep compatible with previous protocol handling and due to constraints in testing environment and knowledge, we still support Kerberos not via the netstream layer but via special extension into the input and output modules. That, too, is unfortunate, but given the current resources at hand, there is no alternative to handling in that way. We would be very interested in moving over Kerberos to a netstream driver and any volunteer would be very welcome. + +\section{Future Development} +This section covers topics that can not currently be developed, but where important thoughts came up in discussions. For obvious reasons, the section has brainstorming character. + +\subsection{Lock-Free Queuing} +On a very busy system, lock contention can limit performance. We should investigate ways to apply lock-free algorithms inside rsyslog. It is believed that at least for some scenarios, lock-free algorigthms can be applied with great benefit. To do so, we should introduce new queue modes, which will use very different semantics from what is described so far for the queue engine. Most importantly, in lock-free mode we will have limits on the number of producers and we will most probably not be able to guarantee audit-grade processing. The later is not a problem, because there are ample use cases that do not require audit-gradeness. + +\subsection{Audit-Grade High Performance Queue Storage Driver} +An audit grade driver must ensure that no message is lost, but should also be able to handle large workloads. The sequential disk driver does not support the later. + +An additional disk driver is envisioned with the properties like the linked list driver, but a reliable on-disk store. In particular, random access to queue elements is desired, which requires an addressing capability. + +A potential implementation requires a pre-formatted file. That file is organized in pages of $n$ bytes (e.g. 1K). The page index is used to address a queue item. If an item fits into 1K, it uses one page. If it is larger than 1K, consequtive pages are used to store the element. A page header must be present to indicate how many pages a single element is made up of. + +It may be noted that we could even improve performance by keeping part of the data in-memory. For audit-gradeness, it is required that upon enqueue the message is written to disk and only after final processing it needs to be removed. However, it is not forbidden to keep the same message in main memory. That way, the logical dequeue operation could be done one the in-memory representation. Only the physical dequeue would need to write to disk again. As such, we save one disk read out of three writes and one read otherwise required (so one can roughly say that we save one third of disk operations. + +Note that due to potential multi-pages messages we can not directly address individual elements, but we can reliably and quikly address elements whom's address we know (learned, for example, during logical dequeue). This is similar to the organization of the in-memory linked list. Actally, such a store \emph{is} a linked list implementation, just that memory is allocated on disk instead of in main memory. + +To further improve speed, object representation could be zipped before being written to a page. + +File Layout +Page 0: control structures (most importantyle queue pointers) (can make sense to store in a separate file, which could be moved to a dedicated disk subsystem - can potentially greatly reduce disk seek times). +Page 1 to n: actual object storage + +Algorithms \ref{alg_AuditGradeStoreEnqueue} and \ref{alg_AuditGradeStoreDelete} show how records are enqueued and deleted. Note that the delete part does not even need to read back the record. If we keep at last some records in-memory, the performance cost of ultra-reliable mode can actually comparatively low. Note that we may not even really need to commit data to the storage system in ``AuditGradeStoreDelete()'', because if a fatal failure occurs at this point, at worst message duplication may happen, what we have considered to be acceptable. + +\begin{algorithm} +\caption{AuditGradeStoreEnqueue($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreEnqueue} +\REQUIRE queue mutex is locked by caller +\STATE write $o$ to current enqueue location +\STATE update \& write queue structures [page 0] +\STATE sync all files touched +\STATE store $o$ in an in-memory structure (or a cache) +\end{algorithmic} +\end{algorithm} + +\begin{algorithm} +\caption{AuditGradeStoreDelete($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreDelete} +\REQUIRE queue mutex is locked by caller +\STATE update queue dequeue pointer \& write queue structures [page 0] +\STATE sync all files touched +\end{algorithmic} +\end{algorithm} + + +\end{document} diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html index cc2f7f38..63c186a3 100644 --- a/doc/dev_oplugins.html +++ b/doc/dev_oplugins.html @@ -144,19 +144,172 @@ array-passing capability not blindly be used.</b> In such cases, we can not guar plugin from segfaulting and if the plugin (as currently always) is run within rsyslog's process space, that results in a segfault for rsyslog. So do not do this. <h3>Batching of Messages</h3> -<p>With the current plugin interface, each message is passed via a separate call to the plugin. -This is annoying and costs performance in some uses cases (primarily for database outputs). -However, that's the way it (currently) is, no easy way around it. There are some ideas -to implement batching capabilities inside the rsyslog core, but without that the only -resort is to do it inside your plugin yourself. You are not prohibited from doing so. -There are some consequences, though: most importantly, the rsyslog core is no longer -intersted in messages that it passed to a plugin. As such, it will not try to make sure -the message is not lost before it was ultimately processed (because rsyslog, due to -doAction() returning successfully, thinks the message *was* ultimately processed). -<p>When the rsyslog core receives batching capabilities, this will be implemented in -a way that is fully compatible to the existing plugin interface. While we have not yet -thought about the implementation, that will probably mean that some new interfaces -or options be used to turn on batching capabilities. +<p>Starting with rsyslog 4.3.x, batching of output messages is supported. Previously, only +a single-message interface was supported. +<p>With the <b>single message</b> plugin interface, each message is passed via a separate call to the plugin. +Most importantly, the rsyslog engine assumes that each call to the plugin is a complete transaction +and as such assumes that messages be properly commited after the plugin returns to the engine. +<p>With the <b>batching</b> interface, rsyslog employs something along the line of +"transactions". Obviously, the rsyslog core can not make non-transactional outputs +to be fully transactional. But what it can is support that the output tells the core which +messages have been commited by the output and which not yet. The core can than take care +of those uncommited messages when problems occur. For example, if a plugin has received +50 messages but not yet told the core that it commited them, and then returns an error state, the +core assumes that all these 50 messages were <b>not</b> written to the output. The core then +requeues all 50 messages and does the usual retry processing. Once the output plugin tells the +core that it is ready again to accept messages, the rsyslog core will provide it with these 50 +not yet commited messages again (actually, at this point, the rsyslog core no longer knows that +it is re-submiting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 +messages were commited (before it failed), then only 10 would have been requeued and resubmitted. +<p>In order to provide an efficient implementation, there are some (mild) constraints in that +transactional model: first of all, rsyslog itself specifies the ultimate transaction boundaries. +That is, it tells the plugin when a transaction begins and when it must finish. The plugin +is free to commit messages in between, but it <b>must</b> commit all work done when the core +tells it that the transaction ends. All messages passed in between a begin and end transaction +notification are called a batch of messages. They are passed in one by one, just as without +transaction support. Note that batch sizes are variable within the range of 1 to a user configured +maximum limit. Most importantly, that means that plugins may receive batches of single messages, +so they are required to commit each message individually. If the plugin tries to be "smarter" +than the rsyslog engine and does not commit messages in those cases (for example), the plugin +puts message stream integrity at risk: once rsyslog has notified the plugin of transacton end, +it discards all messages as it considers them committed and save. If now something goes wrong, +the rsyslog core does not try to recover lost messages (and keep in mind that "goes wrong" +includes such uncontrollable things like connection loss to a database server). So it is +highly recommended to fully abide to the plugin interface details, even though you may +think you can do it better. The second reason for that is that the core engine will +have configuration settings that enable the user to tune commit rate to their use-case +specific needs. And, as a relief: why would rsyslog ever decide to use batches of one? +There is a trivial case and that is when we have very low activity so that no queue of +messages builds up, in which case it makes sense to commit work as it arrives. +(As a side-note, there are some valid cases where a timeout-based commit feature makes sense. +This is also under evaluation and, once decided, the core will offer an interface plus a way +to preserve message stream integrity for properly-crafted plugins). +<p>The second restriction is that if a plugin makes commits in between (what is perfectly +legal) those commits must be in-order. So if a commit is made for message ten out of 50, +this means that messages one to nine are also commited. It would be possible to remove +this restriction, but we have decided to deliberately introduce it to simpify things. +<h3>Output Plugin Transaction Interface</h3> +<p>In order to keep compatible with existing output plugins (and because it introduces +no complexity), the transactional plugin interface is build on the traditional +non-transactional one. Well... actually the traditional interface was transactional +since its introduction, in the sense that each message was processed in its own +transaction. +<p>So the current <code>doAction()</b> entry point can be considered to have this +structure (from the transactional interface point of view): +<p><pre><code> +doAction() + { + beginTransaction() + ProcessMessage() + endTransaction() + } + </code></pre> +<p>For the <b>transactional interface</b>, we now move these implicit <code>beginTransaction()</code> +and <code>endTransaction(()</code> call out of the message processing body, resulting is such +a structure: +<p><pre><code> +beginTransaction() + { + /* prepare for transaction */ + } + +doAction() + { + ProcessMessage() + /* maybe do partial commits */ + } + +endTransaction() + { + /* commit (rest of) batch */ + } +</code></pre> +<p>And this calling structure actually is the transactional interface! It is as simple as this. +For the new interface, the core calls a <code>beginTransaction()</code> entry point inside the +plugin at the start of the batch. Similarly, the core call <code>endTransaction()</code> at the +end of the batch. The plugin must implement these entry points according to its needs. +<p>But how does the core know when to use the old or the new calling interface? This is rather +easy: when loading a plugin, the core queries the plugin for the <code>beginTransaction()</code> +and <code>endTransaction()</code> entry points. If the plugin supports these, the new interface is +used. If the plugin does not support them, the old interface is used and rsyslog implies that +a commit is done after each message. Note that there is no special "downlevel" handling +necessary to support this. In the case of the non-transactional interface, rsyslog considers +each completed call to <code>doAction</code> as partial commit up to the current message. +So implementation inside the core is very straightforward. +<p>Actually, <b>we recommend that the transactional entry points only be defined by those +plugins that actually need them</b>. All others should not define them in which case +the default commit behaviour inside rsyslog will apply (thus removing complexity from the +plugin). +<p>In order to support partial commits, special return codes must be defined for +<code>doAction</code>. All those return codes mean that processing completed successfully. +But they convey additional information about the commit status as follows: +<p> +<table border="0"> +<tr> +<td valign="top"><i>RS_RET_OK</i></td> +<td>The record and all previous inside the batch has been commited. +<i>Note:</i> this definition is what makes integrating plugins without the +transaction being/end calls so easy - this is the traditional "success" return +state and if every call returns it, there is no need for actually calling +<code>endTransaction()</code>, because there is no transaction open).</td> +</tr> +<tr> +<td valign="top"><i>RS_RET_DEFER_COMMIT</i></td> +<td>The record has been processed, but is not yet commited. This is the +expected state for transactional-aware plugins.</td> +</tr> +<tr> +<td valign="top"><i>RS_RET_PREVIOUS_COMMITTED</i></td> +<td>The <b>previous</b> record inside the batch has been committed, but the +current one not yet. This state is introduced to support sources that fill up +buffers and commit once a buffer is completely filled. That may occur halfway +in the next record, so it may be important to be able to tell the +engine the everything up to the previouos record is commited</td> +</tr> +</table> +<p>Note that the typical <b>calling cycle</b> is <code>beginTransaction()</code>, +followed by <i>n</i> times +<code>doAction()</code></n> followed by <code>endTransaction()</code>. However, if either +<code>beginTransaction()</code> or <code>doAction()</code> return back an error state +(including RS_RET_SUSPENDED), then the transaction is considered aborted. In result, the +remaining calls in this cycle (e.g. <code>endTransaction()</code>) are never made and a +new cycle (starting with <code>beginTransaction()</code> is begun when processing resumes. +So an output plugin must expect and handle those partial cycles gracefully. +<p><b>The question remains how can a plugin know if the core supports batching?</b> +First of all, even if the engine would not know it, the plugin would return with RS_RET_DEFER_COMMIT, +what then would be treated as an error by the engine. This would effectively disable the +output, but cause no further harm (but may be harm enough in itself). +<p>The real solution is to enable the plugin to query the rsyslog core if this feature is +supported or not. At the time of the introduction of batching, no such query-interface +exists. So we introduce it with that release. What the means is if a rsyslog core can +not provide this query interface, it is a core that was build before batching support +was available. So the absence of a query interface indicates that the transactional +interface is not available. One might now be tempted the think there is no need to do +the actual check, but is is recommended to ask the rsyslog engine explicitely if +the transactional interface is present and will be honored. This enables us to +create versions in the future which have, for whatever reason we do not yet know, no +support for this interface. +<p>The logic to do these checks is contained in the <code>INITChkCoreFeature</code> macro, +which can be used as follows: +<p><pre><code> +INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); +</code></pre> +<p>Here, bCoreSupportsBatching is a plugin-defined integer which after execution is +1 if batches (and thus the transational interface) is supported and 0 otherwise. +CORE_FEATURE_BATCHING is the feature we are interested in. Future versions of rsyslog +may contain additional feature-test-macros (you can see all of them in +./runtime/rsyslog.h). +<p>Note that the ompsql output plugin supports transactional mode in a hybrid way and +thus can be considered good example code. + +<h2>Open Issues</h2> +<ul> +<li>Processing errors handling +<li>reliable re-queue during error handling and queue termination +</ul> + + + <h3>Licensing</h3> <p>From the rsyslog point of view, plugins constitute separate projects. As such, we think plugins are not required to be compatible with GPLv3. However, this is diff --git a/doc/highperf.txt b/doc/highperf.txt new file mode 100644 index 00000000..5f9481e1 --- /dev/null +++ b/doc/highperf.txt @@ -0,0 +1,4 @@ +links to high performance papers: + +- http://www.kegel.com/c10k.html +- http://pl.atyp.us/content/tech/servers.html (**) diff --git a/doc/imtcp.html b/doc/imtcp.html index 0ccdecc7..422bbd55 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -1,27 +1,29 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html><head> -<meta http-equiv="Content-Language" content="en"><title>TCP Syslog Input Module</title></head> +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>TCP Syslog Input Module</title> +</head> + <body> -<a href="rsyslog_conf_modules.html">back</a> +<a href="rsyslog_conf_modules.html">back to rsyslog module overview</a> <h1>TCP Syslog Input Module</h1> <p><b>Module Name: imtcp</b></p> -<p><b>Author: </b>Rainer Gerhards -<rgerhards@adiscon.com></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Multi-Ruleset Support: </b>since 4.5.0 and 5.1.1 <p><b>Description</b>:</p> <p>Provides the ability to receive syslog messages via TCP. -Encryption can be provided by using <a href="rsyslog_stunnel.html">stunnel</a> -(an alternative is the use -the <a href="imgssapi.html">imgssapi</a> -modul).</p> -<p>Multiple receivers may be configured by -specifying +Encryption is natively provided by selecting the approprioate network stream driver and +can also be provided by using <a href="rsyslog_stunnel.html">stunnel</a> +(an alternative is the use the <a href="imgssapi.html">imgssapi</a> module).</p> +<p>Multiple receivers may be configured by specifying $InputTCPServerRun multiple times. This is available since version 4.3.1, earlier versions do NOT support it. </p> <p><b>Configuration Directives</b>:</p> <ul> -<li>$InputTCPServerAddtlFrameDelimiter <Delimiter><br> +<li><b>$InputTCPServerAddtlFrameDelimiter <Delimiter></b><br> This directive permits to specify an additional frame delimiter for plain tcp syslog. The industry-standard specifies using the LF character as frame delimiter. Some vendors, notable Juniper in their NetScreen products, use an invalid frame delimiter, in Juniper's @@ -41,28 +43,37 @@ very limited interest in fixing this issue. This directive <b>can not</b> fix th That would require much more code changes, which I was unable to do so far. Full details can be found at the <a href="http://www.rsyslog.com/Article321.phtml">Cisco tcp syslog anomaly</a> page. -<li>$InputTCPServerNotifyOnConnectionClose [on/<b>off</b>] (available since 4.5.5)<br> +<li><b>$InputTCPServerDisableLFDelimiter</b> <on/<b>off</b>> (available since 5.5.3)<br> +Industry-strandard plain text tcp syslog uses the LF to delimit syslog frames. However, +some users brought up the case that it may be useful to define a different delimiter and +totally disable LF as a delimiter (the use case named were multi-line messages). This mode +is non-standard and will probably come with a lot of problems. However, as there is need +for it and it is relatively easy to support, we do so. Be sure to turn this setting to +"on" only if you exactly know what you are doing. You may run into all sorts of troubles, +so be prepared to wrangle with that! +<li><b>$InputTCPServerNotifyOnConnectionClose</b> [on/<b>off</b>] (available since 4.5.5)<br> instructs imtcp to emit a message if the remote peer closes a connection.<br> <b>Important:</b> This directive is global to all listeners and must be given right after loading imtcp, otherwise it may have no effect.</li> -<li>$InputTCPServerRun <port><br> +<li><b>$InputTCPServerRun</b> <port><br> Starts a TCP server on selected port</li> -<li>$InputTCPMaxListeners <number><br> +<li><b>$InputTCPMaxListeners</b> <number><br> Sets the maximum number of listeners (server ports) supported. Default is 20. This must be set before the first $InputTCPServerRun directive.</li> -<li>$InputTCPMaxSessions <number><br> -Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> -<li>$InputTCPServerStreamDriverMode <number><br> +<li><b>$InputTCPMaxSessions</b> <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> +<li><b>$InputTCPServerStreamDriverMode</b> <number><br> Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specifc.</li> -<li>$InputTCPServerInputName <name><br> +<li><b>$InputTCPServerInputName</b> <name><br> Sets a name for the inputname property. If no name is set "imtcp" is used by default. Setting a name is not strictly necessary, but can be useful to apply filtering based on which input the message was received from. -<li>$InputTCPServerStreamDriverAuthMode <mode-string><br> +<li><b>$InputTCPServerStreamDriverAuthMode</b> <mode-string><br> Sets the authentication mode for the currently selected <a href="netstream.html">network stream driver</a>. <mode-string> is driver specifc.</li> -<li>$InputTCPServerStreamDriverPermittedPeer <id-string><br> +<li><b>$InputTCPServerStreamDriverPermittedPeer</b> <id-string><br> Sets permitted peer IDs. Only these peers are able to connect to the listener. <id-string> semantics depend on the currently selected AuthMode and <a href="netstream.html">network stream driver</a>. PermittedPeers may not be set in anonymous modes.</li> +<li><b>$InputTCPServerBindRuleset</b> <ruleset><br> +Binds the listener to a specific <a href="multi_ruleset.html">ruleset</a>.</li> </ul> <b>Caveats/Known Bugs:</b> <ul> @@ -70,20 +81,22 @@ AuthMode and <a href="netstream.html">network stream driver</a>. Permitted <li>can not be loaded together with <a href="imgssapi.html">imgssapi</a> (which includes the functionality of imtcp)</li> </ul> -<p><b>Sample:</b></p> -<p>This sets up a TCP server on port 514:<br> +<p><b>Example:</b></p> +<p>This sets up a TCP server on port 514 and permits it to accept up to 500 connections:<br> </p> -<textarea rows="15" cols="60">$ModLoad imtcp # -needs to be done just once +<textarea rows="15" cols="60">$ModLoad imtcp # needs to be done just once +$InputTCPMaxSessions 500 $InputTCPServerRun 514 </textarea> +<p>Note that the parameters (here: max sessions) need to be set <b>before</b> the listener +is activated. Otherwise, the parameters will not apply. +</p> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> -<p><font size="2">This documentation is part of the -<a href="http://www.rsyslog.com/">rsyslog</a> +<p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +Copyright © 2008,2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html> +</body> +</html> diff --git a/doc/imudp.html b/doc/imudp.html new file mode 100644 index 00000000..f0e86307 --- /dev/null +++ b/doc/imudp.html @@ -0,0 +1,58 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> +<meta http-equiv="Content-Language" content="en"> +<title>TCP Syslog Input Module</title> +</head> + +<body> +<a href="rsyslog_conf_modules.html">back to rsyslog module overview</a> + +<h1>UDP Syslog Input Module</h1> +<p><b>Module Name: imudp</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Multi-Ruleset Support: </b>since 5.3.2 +<p><b>Description</b>:</p> +<p>Provides the ability to receive syslog messages via UDP. +<p>Multiple receivers may be configured by specifying +$UDPServerRun multiple times. +</p> +<p><b>Configuration Directives</b>:</p> +<ul> +<li>$UDPServerAddress <IP><br> +local IP address (or name) the UDP listens should bind to</li> +<li>$UDPServerRun <port><br> +former -r<port> option, default 514, start UDP server on this +port, "*" means all addresses</li> +<li>$UDPServerTimeRequery <nbr-of-times><br> +this is a performance +optimization. Getting the system time is very costly. With this setting, imudp can +be instructed to obtain the precise time only once every n-times. This logic is +only activated if messages come in at a very fast rate, so doing less frequent +time calls should usually be acceptable. The default value is two, because we have +seen that even without optimization the kernel often returns twice the identical time. +You can set this value as high as you like, but do so at your own risk. The higher +the value, the less precise the timestamp. +<li>$InputUDPServerBindRuleset <ruleset><br> +Binds the listener to a specific <a href="multi_ruleset.html">ruleset</a>.</li> +</ul> +<b>Caveats/Known Bugs:</b> +<ul> +<li>currently none known</li> +</ul> +<p><b>Sample:</b></p> +<p>This sets up an UPD server on port 514:<br> +</p> +<textarea rows="15" cols="60">$ModLoad imudp # needs to be done just once +$UDPServerRun 514 +</textarea> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer +Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/index.html b/doc/index.html index b3b336a7..d753e2ed 100644 --- a/doc/index.html +++ b/doc/index.html @@ -13,7 +13,7 @@ installed documentation which exactly matches the version you have installed. It is highly suggested to at least briefly look over these files. <li>The <a href="http://www.rsyslog.com">rsyslog web site</a> which offers probably every information you'll ever need (ok, just kidding...). -<li>The <a href="http://www.rsyslog.com/doc-status.html">project status page</a> provides +<li>The <a href="http://www.rsyslog.com/status">project status page</a> provides information on current releases <li>and the <a href="troubleshoot.html">troubleshooting guide</a> hopefully helps if things do not immediately work out diff --git a/doc/manual.html b/doc/manual.html index 0e4166d0..882af031 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,8 +19,8 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 4.7.2 (v4-devel branch) of rsyslog.</b> -Visit the <i> <a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> +<p><b>This documentation is for version 5.5.7 (beta branch) of rsyslog.</b> +Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might want to lend us a helping hand. </b>It doesn't require a lot of @@ -28,9 +28,11 @@ time - even a single mouse click helps. Learn <a href="how2help.html">how to hel Due to popular demand, there is now a <a href="rsyslog_ng_comparison.html">side-by-side comparison between rsyslog and syslog-ng</a>.</p> <p>If you are upgrading from rsyslog v2 or stock sysklogd, -<a href="v3compatibility.html">be sure to read the rsyslog v3 compatibility document</a>, +<a href="v3compatibility.html">be sure to read the rsyslog v3 compatibility notes</a>, and if you are upgrading from v3, read the -<a href="v4compatibility.html">rsyslog v4 compatibility document</a>. +<a href="v4compatibility.html">rsyslog v4 compatibility notes</a> and +if you upgrade from v4, read the +<a href="v5compatibility.html">rsyslog v5 compatibility notes</a>. <p>Rsyslog will work even if you do not read the doc, but doing so will definitely improve your experience.</p> <p><b>Follow the links below for the</b></p> @@ -41,10 +43,10 @@ if you do not read the doc, but doing so will definitely improve your experience <li> <a href="property_replacer.html">property replacer, an important core component</a></li> <li>a commented <a href="sample.conf.html">sample rsyslog.conf</a> </li> <li><a href="bugs.html">rsyslog bug list</a></li> -<li><a href="rsyslog_packages.html"> rsyslog packages</a></li> -<li><a href="generic_design.html">backgrounder on -generic syslog application design</a> +<li><a href="messageparser.html">understanding rsyslog message parsers</a></li> +<li><a href="generic_design.html">backgrounder on generic syslog application design</a></li> <li><a href="modules.html">description of rsyslog modules</a></li> +<li><a href="rsyslog_packages.html">rsyslog packages</a></li> <li><a href="http://cookbook.rsyslog.com">the rsyslog "cookbook"</a> - a set of configurations ready to use</li> </ul> <p><b>We have some in-depth papers on</b></p> @@ -69,6 +71,7 @@ syslog sender over NAT</a> (online only)</li> <li><a href="debug.html">debug support in rsyslog</a></li> <li>Developer Documentation <ul> + <li><a href="build_from_repo.html">building rsyslog from the source repository</a></li> <li><a href="dev_oplugins.html">writing rsyslog output plugins</a></li> <li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> </ul></li> diff --git a/doc/messageparser.html b/doc/messageparser.html new file mode 100644 index 00000000..370db59f --- /dev/null +++ b/doc/messageparser.html @@ -0,0 +1,222 @@ +<html> +<head> +<title>Message parsers in rsyslog</title> +</head> +<body> +<a href="manual.html">rsyslog documentation</a> + +<h1>Message parsers in rsyslog</h1> +<p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> +(2009-11-06)</i></small></p> +<h2>Intro</h2> +<p>Message parsers are a feature of rsyslog 5.3.4 and above. In this article, I describe what +message parsers are, what they can do and how they relate to the relevant standards. I will +also describe what you can not do with time. Finally, I give some advice on implementing your +own custom parser. + +<h2>What are message parsers?</h2> +<p>Well, the quick answer is that message parsers are the component of rsyslog that +parses the syslog message after it is being received. Prior to rsyslog 5.3.4, message parsers +where built in into the rsyslog core itself and could not be modified (other than by modifying +the rsyslog code). +<p>In 5.3.4, we changed that: message parsers are now loadable modules (just +like input and output modules). That means that new message parsers can be added without +modifying the rsyslog core, even without contributing something back to the +project. +<p>But that doesn't answer what a message parser really is. What does ist mean to "parse a +message" and, maybe more importantly, what is a message? To answer these questions correctly, +we need to dig down into the relevant standards. +<a href="http://tools.ietf.org/html/rfc5424">RFC5424</a> specifies a layered architecture +for the syslog protocol: +<p align="center"><img src="rfc5424layers.png" alt="RFC5424 syslog protocol layers"> +<p>For us important is the distinction between the syslog transport and the upper layers. +The transport layer specifies how a stream of messages is assembled at the sender side and +how this stream of messages is disassembled into the individual messages at the receiver +side. In networking terminology, this is called "framing". The core idea is that +each message is put into a so-called "frame", which then is transmitted over the communications +link. +<p>The framing used is depending on the protocol. For example, in UDP the "frame"-equivalent is +a packet that is being sent (this also means that no two messages can travel within a single +UDP packet). In "plain tcp syslog", the industry standard, LF is used as a frame delimiter +(which also means that no multi-line message can properly be transmitted, a "design" flaw +in plain tcp syslog). In <a href="http://tools.ietf.org/html/rfc5425">RFC5425</a> there is +a header in front of each frame that contains the size of the message. With this framing, +any message content can properly be transferred. +<p>And now comes the important part: <b>message parsers do NOT operate at the transport +layer</b>, they operate, as their name implies, on messages. So we can not use message +parsers to change the underlying framing. For example, if a sender splits (for whatever +reason) a single message into two and encapsulates these into two frames, there is no way +a message parser could undo that. +<p>A typical example may be a multi-line message: let's assume some originator has generated +a message for the format "A\nB" (where \n means LF). If that message is being transmitted +via plain tcp syslog, the frame delimiter is LF. So the sender will delimite the frame with +LF, but otherwise send the message unmodified onto the wire (because that is how things are +-unfortunately- done in plain tcp syslog...). So wire will see "A\nB\n". When this +arrives at the receiver, the transport layer will undo the framing. When it sees the LF +after A, it thinks it finds a valid frame delimiter (in fact, this is the correct view!). So +the receive will extract one complete message A and one complete message B, not knowing +that they once were both part of a large multi-line message. These two messages are then +passed to the upper layers, where the message parsers receive them and extract information. +However, the message parsers never know (or even have a chance to see) that A and B +belonged together. Even further, in rsyslog there is no guarnatee that A will be parsed +before B - concurrent operations may cause the reverse order (and do so very validly). +<p>The important lesson is: <b>message parsers can not be used to fix a broken framing</b>. +You need a full protocol implementation to do that, what is the domain of input and +output modules. +<p>I have now told you what you can not do with message parsers. But what they are good for? +Thankfully, broken framing is not the primary problem of the syslog world. A wealth of different +formats is. Unfortunately, many real-world implementations violate the relevant standards +in one way or another. That makes it often very hard to extract meaningful information from +a message or to process messages from different sources by the same rules. In my article +<a href="syslog_parsing.html">syslog parsing in rsyslog</a> I have elaborated on all +the real-world evil that you can usually see. So I won't repeat that here. But in short, the +real problem is not the framing, but how to make malformed messages well-looking. +<p><b>This is what message parsers permit you to do: take a (well-known) malformed message, parse +it according to its semantics and generate perfectly valid internal message representations +from it.</b> So as long as messages are consistenly in the same wrong format (and they usually +are!), a message parser can look at that format, parse it, and make the message processable just +like it were wellformed in the first place. Plus, one can abuse the interface to do some other +"intersting" tricks, but that would take us to far. +<p>While this functionality may not sound exciting, it actually solves a very big issue (that you +only really understand if you have managed a system with various different syslog sources). +Note that we were often able to process malformed messages in the past with the help of the +property replacer and regular expressions. While this is nice, it has a performance hit. A +message parser is a C code, compiled to native language, and thus typically much faster than +any regular expression based method (depending, of course, on the quality of the implementation...). + +<h2>How are message parsers used?</h2> +<p>In a simlified view, rsyslog +<ol> +<li>first receives messages (via the input module), +<li><i>then parses them (at the message level!)</i> and +<li>then processes them (operating on the internal message representation). +</ol> +Message parsers are utilized in the second step (written in italics). +Thus, they take the raw message (NOT frame!) received from the remote system and create +the internal structure out of it that the other parts of rsyslog need in order to perform +their processing. Parsing is vital, because an unparsed message can not be processed in the +third stage, the actual application-level processing (like forwarding or writing to files). +<h3>Parser Chains and how they Operate</h3> +Rsyslog chains parsers together to provide flexibility. +A <b>parser chain</b> +contains all parsers that can potentially be used to parse a message. +It is assumed that there is some +way a parser can detect if the message it is being presented is supported by it. If so, the parser +will tell the rsyslog engine and parse the message. The rsyslog engine now calls each parser +inside the chain (in sequence!) until the first parser is able to parse the message. After one +parser has been found, the message is considered parsed and no others parsers are called on that +message. +<p>Side-note: this method implies there are some "not-so-dirty" tricks available to modify +the message by a parser module that declares itself as "unable to parse" but still does +some message modification. This was not a primary design goal, but may be utilized, and the +interface probably extended, to support generic filter modules. These would need to go +to the root of the parser chain. As mentioned, the current system already supports this. +<p>The position inside the parser chain can be thought of as a priority: parser sitting +earlier in the chain take precedence over those sitting later in it. So more specific +parser should go ealier in the chain. A good example of how this works is the default parser +set provided by rsyslog: rsyslog.rfc5424 and rsyslog.rfc3164, each one parses according to the +rfc that has named it. RFC5424 was designed to be distinguishable from RFC3164 message by the +sequence "1 " immediately after the so-called PRI-part (don't worry about these words, it is +sufficient if you understand there is a well-defined sequence used to indentify RFC5424 +messages). In contrary, RFC3164 actually permits everything as a valid message. Thus the +RFC3164 parser will always parse a message, sometimes with quite unexpected outcome (there is +a lot of guesswork involved in that parser, which unfortunately is unavoidable due to +existing techology limits). So the default parser chain is to try the RFC5424 parser first +and after it the RFC3164 parser. If we have a 5424-formatted message, that parser will +identify and parse it and the rsyslog engine will stop processing. But if we receive a +legacy syslog message, the RFC5424 will detect that it can not parse it, return this status +to the engine which then calls the next parser inside the chain. That usually happens to be +the RFC3164 parser, which will always process the message. But there could also be any other +parser inside the chain, and then each one would be called unless one that is able to parse +can be found. +<p>If we reversed the parser order, RFC5424 messages would incorrectly parsed. Why? Because the +RFC3164 parser will always parse every message, so if it were asked first, it would parse +(and misinterpret) the 5424-formatted message, return it did so and the rsyslog engine would +never call the 5424 parser. So oder of sequence is very important. +<p>What happens if no parser in the chain could parse a message? Well, then we could not +obtain the in-memory representation that is needed to further process the message. In that +case, rsyslog has no other choice than to discard the message. If it does so, it will emit +a warning message, but only in the first 1,000 incidents. This limit is a safety measure +against message-loops, which otherwise could quickly result from a parser chain +misconfiguration. <b>If you do not tolerate loss of unparsable messages, you must ensure +that each message can be parsed.</b> You can easily achive this by always using the +"rsyslog-rfc3164" parser as the <i>last</i> parser inside parser chains. That may result +in invalid parsing, but you will have a chance to see the invalid message (in debug mode, +a warning message will be written to the debug log each time a message is dropped due to +inability to parse it). +<h3>Where are parser chains used?</h3> +<p>We now know what parser chains are and how they operate. The question is now how many +parser chains can be active and how it is decicded which parser chain is used on which message. +This is controlled via <a href="multi_ruleset.html">rsyslog's rulesets</a>. In short, multiple +rulesets can be defined and there always exist at least one ruleset (for specifcs, follow +the <a href="multi_ruleset.html">link</a>). A parser chain is bound to a specific ruleset. +This is done by virtue of defining parsers via the +<a href="rsconf1_rulesetparser.html">$RulesetParser</a> configuration directive (for specifics, +see there). If no such directive is specified, the default parser chain is used. As of this +writing, the default parser chain always consists of "rsyslog.rfc5424", "rsyslog.rfc3164", in +that order. As soon as a parser is configured, the default list is cleared and the new parser +is added to the end of the (initially empty) ruleset's parser chain. +<p>The important point to know is that parser chains are defined on a per-ruleset basis. +<h3>Can I use different parser chains for different devices?</h3> +<p>The correct answer is: generally yes, but it depends. First of all, remember that input +modules (and specific listeners) may be bound to specific rulesets. As parser chains "reside" +in rulesets, binding to a ruleset also binds to the parser chain that is bound to that ruleset. +As a number one prequisite, the input module must support binding to different rulesets. Not +all do, but their number is growing. For example, the important +<a href="imudp.html">imudp</a> and <a href="imtcp.html">imtcp</a> input modules support +that functionality. Those that do not (for example <a href="im3195">im3195</a>) can only +utilize the default ruleset and thus the parser chain defined in that ruleset. +<p>If you do not know if the input module in question supports ruleset binding, check +its documentation page. Those that support it have the requiered directives. +<p>Note that it is currently under evaluation if rsyslog will support binding parser chains +to specific inputs directly, without depending on the ruleset. There are some concerns that +this may not be necessary but adds considerable complexity to the configuration. So this may +or may not be possible in the future. In any case, if we decide to add it, input modules +need to support it, so this functionality would require some time to implement. +<p>The coockbook recipe for using different parsers for different devices is given +as an actual in-depth example in the <a href="rscon1_rulesetsparser.html">$RulesetParser</a> +configuration directive doc page. In short, it is acomplished by defining specific rulesets +for the required parser chains, definining different listener ports for each of the devices +with different format and binding these listeners to the correct ruleset (and thus parser +chains). Using that approach, a variety of different message formats can be supported +via a single rsyslog instance. + +<h2>Which message parsers are available</h2> +<p>As of this writing, there exist only two message parsers, one for RFC5424 format and one for +legacy syslog (loosely described in +<a href="http://tools.ietf.org/html/rfc3164">RFC3164</a>). These parsers are built-in and +must not be explicitely loaded. However, message parsers can be added with relative ease +by anyone knowing to code in C. Then, they can be loaded via $ModLoad just like any +other loadable module. It is expected that the rsyslog project will be contributed additional +message parsers over time, so that at some point there hopefully is a rich choice of them +(I intend to add a browsable repository as soon as new parsers pop up). +<h3>How to write a message parser?</h3> +<p>As a prequisite, you need to know the exact format that the device is sending. Then, you need +moderate C coding skills, and a little bit of rsyslog internals. I guess the rsyslog specific part +should not be that hard, as almost all information can be gained from the existing parsers. They +are rather simple in structure and can be found under the "./tools" directory. They are named +pmrfc3164.c and pmrfc5424.c. You need to follow the usual loadable module guidelines. +It is my expectation that writing a parser should typically not take longer than a single +day, with maybe a day more to get aquainted with rsyslog. Of course, I am not sure if the number +is actually right. +<p>If you can not program or have no time to do it, Adiscon can also write a message parser +for you as +part of the <a href="http://www.rsyslog/professional-services">rsyslog professional services +offering</a>. +<h2>Conclusion</h2> +<p>Malformed syslog messages are a pain and unfortunately often seen in practice. Message parsers +provide a fast and efficient solution for this problem. Different parsers can be defined for +different devices, and they all convert message information into rsyslog's well-defined +internal format. Message parsers were first introduced in rsyslog 5.3.4 and also offer +some interesting ideas that may be explored in the future - up to full message normalization +capabilities. It is strongly recommended that anyone with a heterogenous environment take +a look at message parser capabilities. + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/module_workflow.png b/doc/module_workflow.png Binary files differnew file mode 100644 index 00000000..e1a72e96 --- /dev/null +++ b/doc/module_workflow.png diff --git a/doc/msgflow.txt b/doc/msgflow.txt new file mode 100644 index 00000000..ebee18f8 --- /dev/null +++ b/doc/msgflow.txt @@ -0,0 +1,56 @@ +flow of messages (in terms of functions) after they have +been pulled off the main queue. + +Functions are listed in the order they are (usually) called +if there are branches in processing flow, this is explicitely +stated. + +as of: 2010-06-08, master branch (v5) + +syslogd.c/msgConsumer +syslogd.c/msgConsumeOne + if ACLcheck needed: + net.cvthname, + net.isAllowedSinder2 + MsgSetRcvFromStr + MsgSetRcvFromIPStr + if NEEDS_PARSING: + parser.ParseMsg +ruleset.ProcessBatch (loops through ruleset) +ruleset.c/processMsgDoRules (for each rule in ruleset) +rule.c/processMsg +1:rule.c/shouldProcessThisMessage + (evaluates filters, optimize via ALL-Filter) +if to be processed, loop through associated actions -> +2:rule.c/processMsgsDoAction +action.c/actionCallAction (LOCKs action object!) +action.c/doActionCallAction (does duplicate message reduction) +action.c/actionWriteToAction + limits based on iExecEveryNthOccur + generates "message repeated..." string if necessary + limits based on iSecsExecOnceInterval +! **qqueueEnqObj** + This means, we are done processing the action at this + stage. The queue may run async, but usually does not + do so (in default settings). + + +Now looking at processing of the action queue. If the queue is +in direct mode, remember that the action object is still +be locked (this may also be a potential bug in non-direct mode, as +it looks like we need this prequisite!). + +action.c/processBatchMain (queue Consumer, LOOK mutActExec) +action.c/processAction + (calls finishBatch at the end, but not so important + for our analysis) +action.c/submitBatch (recursive submit/retry loop for messages) +action.c/tryDoAction (submits a [potentially partial] batch) +action.c/actionProcessMessage + (action.c/actionPrepare (utility to set status/TX mode)) +action.c/actionCallDoAction +1: action.c/prepareDoActionParams +1: template.c/tplToString-tplToArray + string buffer is cached in action object +2:<output Module>/doAction + diff --git a/doc/multi_ruleset.html b/doc/multi_ruleset.html index 8d8c614f..da65b4ba 100644 --- a/doc/multi_ruleset.html +++ b/doc/multi_ruleset.html @@ -23,7 +23,7 @@ write it to a file or forward it to a remote logging server. <p>A traditional configuration file is made up of one or more of these rules. When a new message arrives, its processing starts with the first rule (in order of appearance in rsyslog.conf) and continues for each rule until either all rules have been processed or -a so-called "e;discard" action happens, in which case processing stops and the +a so-called "discard" action happens, in which case processing stops and the message is thrown away (what also happens after the last rule has been processed). <p>The <b>multi-ruleset</b> support now permits to specify more than one such rule sequence. @@ -66,11 +66,18 @@ to seperate the messages by any other method. <pre>$InputTCPServerBindRuleset <name> </pre> -directive. Note that "name"e; must be the name of a ruleset that is already defined +directive. Note that "name" must be the name of a ruleset that is already defined at the time the bind directive is given. There are many ways to make sure this happens, but I personally think that it is best to define all rule sets at the top of rsyslog.conf and define the inputs at the bottom. This kind of reverses the traditional recommended ordering, but seems to be a really useful and straightforward way of doing things. +<h2>Why are rulesets important for different parser configurations?</h2> +<p>Custom message parsers, used to handle differnet (and potentially otherwise-invalid) +message formats, can be bound to rulesets. So multiple rulesets can be a very useful +way to handle devices sending messages in different malformed formats in a consistent +way. Unfortunately, this is not uncommon in the syslog world. An in-depth explanation +with configuration sample can be found at the +<a href="rsconf1_rulesetparser.html">$RulesetParser</a> configuration directive. <h2>Can I use a different Ruleset as the default?</h2> <p>This is possible by using the @@ -249,11 +256,12 @@ $InputTCPServerBindRuleset remote10516 $InputTCPServerRun 10516 </pre> -<p>Note that the "mail.*" rule inside the "remote10516"e; ruleset does +<p>Note that the "mail.*" rule inside the "remote10516" ruleset does not affect processing inside any other rule set, including the default rule set. <h2>Performance</h2> +<h3>Fewer Filters</h3> <p>No rule processing can be faster than not processing a rule at all. As such, it is useful for a high performance system to identify disjunct actions and try to split these off to different rule sets. In the example section, we had a case where three different tcp listeners @@ -263,6 +271,25 @@ is no need to check the reception service - instead messages are automatically p right rule set and can be processed by very simple rules (maybe even with "*.*"-filters, the fastest ones available). +<h3>Partitioning of Input Data</h3> +<p>Starting with rsyslog 5.3.4, rulesets permit higher concurrency. They offer the ability +to run on their own "main" queue. What that means is that a own queue is associated +with a specific rule set. That means that inputs bound to that ruleset do no longer need +to compete with each other when they enqueue a data element into the queue. Instead, enqueue +operations can be completed in parallel. +<p>An example: let us assume we have three TCP listeners. Without rulesets, each of them +needs to insert messages into the main message queue. So if each of them wants to +submit a newly arrived message into the queue at the same time, only one can do so +while the others need to wait. With multiple rulesets, its own queue can be created for each +ruleset. If now each listener is bound to its own ruleset, concurrent message submission is +possible. On a machine with a sufficiently large number of corse, this can result in +dramatic performance improvement. +<p>It is highly advised that high-performance systems define a dedicated ruleset, with a +dedicated queue for each of the inputs. +<p>By default, rulesets do <b>not</b> have their own queue. It must be activated via the +<a href="rsconf1_rulesetcreatemainqueue.html">$RulesetCreateMainQueue</a> directive. + +<h3>Future Enhancements</h3> <p>In the long term, multiple rule sets will probably lay the foundation for even better optimizations. So it is not a bad idea to get aquainted with them. diff --git a/doc/omruleset.html b/doc/omruleset.html new file mode 100644 index 00000000..41d6ccfc --- /dev/null +++ b/doc/omruleset.html @@ -0,0 +1,140 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>ruleset output module (omruleset)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>ruleset output/including module (omruleset)</h1> +<p><b>Module Name: omruleset</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 5.3.4</p> +<p><b>Description</b>:</p> +<p>This is a very special "output" module. It permits to pass a message object +to another rule set. While this is a very simple action, it enables very +complex configurations, e.g. it supports high-speed "and" conditions, sending +data to the same file in a non-racy way, include-ruleset functionality as well as +some high-performance optimizations (in case the rule sets have the necessary +queue definitions). +<p>While it leads to a lot of power, this output module offers seamingly easy functionaltiy. +The complexity (and capablities) arise from how everthing can be combined. +<p>With this module, a message can be sent to processing to another ruleset. This is somewhat +similar to a "#include" in the C programming language. However, one needs to keep +on the mind that a ruleset can contain its own queue and that a queue can run in various modes. +<p>Note that if no queue is defined in the ruleset, the message is enqueued into the main message +queue. This most often is not optimal and means that message processing may be severely defered. +Also note that when the ruleset's target queue is full and no free space can be aquired +within the usual timeout, the message object may actually be lost. This is an extreme scenario, +but users building an audit-grade system need to know this restriction. For regular installations, +it should not really be relevant. +<p><b>At minimum, be sure you understand the +<a href="rsconf1_rulesetcreatemainqueue.html">$RulesetCreateMainQueue</a> +directive as well as the importance of statement order in rsyslog.conf before using omruleset!</b> +<p><b>Recommended Use:</b> +<ul> +<li>create rulesets specifically for omruleset +<li>create these rulesets with their own main queue +<li> decent queueing parameters (sizes, threads, etc) should be used +for the ruleset main queue. If in doubt, use the same parameters as for the +overall main queue. +<li>if you use multiple levels of ruleset nesting, double check for endless loops - the rsyslog +engine does not detect these +</ul> + +<p><b>Configuration Directives</b>:</p> +<ul> +<li><b>$ActionOmrulesetRulesetName</b> ruleset-to-submit-to<br> +This directive specifies the name of the ruleset that the message +provided to omruleset should be submitted to. This ruleset must already have +been defined. Note that the directive is automatically reset after each +:omruleset: action and there is no default. This is done to prevent accidential +loops in ruleset definition, what can happen very quickly. +The :omruleset: action will NOT be honored if no ruleset name has been defined. As usual, +the ruleset name must be specified in front of the action that it modifies. +</ul> +<p><b>Examples:</b></p> +<p>This example creates a ruleset for a write-to-file action. The idea here +is that the same file is written based on multiple filters, problems occur if the file is used +together with a buffer. That is because file buffers are action-specific, and so some partial +buffers would be written. With omruleset, we create a single action inside its own ruleset and +then pass all messages to it whenever we need to do so. Of course, such a simple situation could +also be solved by a more complex filter, but the method used here can also be utilized in +more complex scenarios (e.g. with multiple listeners). The example tries to keep it simple. +Note that we create a ruleset-specific main queue (for simplicity with the default main queue +parameters) in order to avoid re-queueing messages back into the main queue. +</p> +<textarea rows="30" cols="80">$ModLoad omruleset + +# define ruleset for commonly written file +$RuleSet commonAction +$RulesetCreateMainQueue on +*.* /path/to/file.log + +#switch back to default ruleset +$ruleset RSYSLOG_DefaultRuleset + +# begin first action +# note that we must first specify which ruleset to use for omruleset: +$ActionOmrulesetRulesetName CommonAction +mail.info :omruleset: +#end first action + +# begin second action +# note that we must first specify which ruleset to use for omruleset: +$ActionOmrulesetRulesetName CommonAction +:FROMHOST, isequal, "myhost.example.com" :omruleset: +#end second action + +# of course, we can have "regular" actions alongside :omrulset: actions +*.* /path/to/general-message-file.log +</textarea> +<p>The next example is used to creat a high-performance nested and filter condition. Here, +it is first checked if the message contains a string "error". If so, the message is forwarded +to another ruleset which then applies some filters. The advantage of this is that we can use +high-performance filters where we otherwise would need to use the (much slower) expression-based +filters. Also, this enables pipeline processing, in that second ruleset is executed in +parallel to the first one.</p> +<textarea rows="30" cols="80">$ModLoad omruleset + +# define "second" ruleset +$RuleSet nested +$RulesetCreateMainQueue on # again, we use our own queue +mail.* /path/to/mailerr.log +kernel.* /path/to/kernelerr.log +auth.* /path/to/autherr.log + +#switch back to default ruleset +$ruleset RSYSLOG_DefaultRuleset + +# begin first action - here we filter on "error" +# note that we must first specify which ruleset to use for omruleset: +$ActionOmrulesetRulesetName nested +:msg, contains, "error :omruleset: +#end first action + +# begin second action - as an example we can do anything else in +# this processing. Note that these actions are processed concurrently +# to the ruleset "nested" +:FROMHOST, isequal, "myhost.example.com" /path/to/host.log +#end second action + +# of course, we can have "regular" actions alongside :omrulset: actions +*.* /path/to/general-message-file.log +</textarea> +<p><b>Caveats/Known Bugs:</b> +<p>The current configuration file language is not really adequate for a complex construct +like omruleset. Unfortunately, more important work is currently preventing me from redoing the +config language. So use extreme care when nesting rulesets and be sure to test-run your +config before putting it into production, ensuring you have a suffciently large probe +of the traffic run over it. If problems arise, the +<a href="troubleshoot.html">rsyslog debug log</a> is your friend. +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/omstdout.html b/doc/omstdout.html new file mode 100644 index 00000000..0bd10cfb --- /dev/null +++ b/doc/omstdout.html @@ -0,0 +1,42 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>stdout output module (omstdout)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>stdout output module (stdout)</h1> +<p><b>Module Name: omstdout</b></p> +<p><b>Author: </b>Rainer Gerhards +<rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 4.1.6</p> +<p><b>Description</b>:</p> +<p>This module writes any messages that are passed to it to stdout. +It was developed for the rsyslog test suite. However, there may +(limited) other uses exists. Please not that we do not put too much +effort into the quality of this module as we do not expect it to +be used in real deployments. If you do, please drop us a note so +that we can enhance its priority! +<p><b>Configuration Directives</b>:</p> +<ul> +<li><b>$ActionOMStdoutArrayInterface</b> [on|<b>off</b><br> +This setting instructs omstdout to use the alternate +array based method of parameter passing. If used, the values +will be output with commas between the values but no other padding bytes. +This is a test aid for the alternate calling interface. +<li><b>$ActionOMStdoutEnsureLFEnding</b> [<b>on</b>|off<br> +Makes sure that each message is written with a terminating LF. This is needed for +the automatted tests. If the message contains a trailing LF, none is added. +</ul> +<b>Caveats/Known Bugs:</b> +<p>Currently none known. +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/omudpspoof.html b/doc/omudpspoof.html new file mode 100644 index 00000000..16cb9b13 --- /dev/null +++ b/doc/omudpspoof.html @@ -0,0 +1,92 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<title>UDP spoofing output module (omudpspoof)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>UDP spoofing output module (omudpspoof)</h1> +<p><b>Module Name: omstdout</b></p> +<p><b>Author: </b>David Lang <david@lang.hm> and Rainer Gerhards +<rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 5.1.3</p> +<p><b>Description</b>:</p> +<p>This module is similar to the regular UDP forwarder, but permits to +spoof the sender address. Also, it enables to circle through a number of +source ports. +<p><b>Configuration Directives</b>:</p> +<ul> +<li><b>$ActionOMUDPSpoofSourceNameTemplate</b> <templatename><br> +This is the name of the template that contains a +numerical IP address that is to be used as the source system IP address. +While it may often be a constant value, it can be generated as usual via the +property replacer, as long as it is a valid IPv4 address. If not specified, the +build-in default template RSYSLOG_omudpspoofDfltSourceTpl is used. This template is defined +as follows:<br> +$template RSYSLOG_omudpspoofDfltSourceTpl,"%fromhost-ip%"<br> +So in essence, the default template spoofs the address of the system the message +was received from. This is considered the most important use case. +<li><b>$ActionOMUDPSpoofTargetHost</b> <hostname><br> +Host that the messages shall be sent to. +<li><b>$ActionUDPSpoofTargetPort</b> <port><br> +Remote port that the messages shall be sent to. +<li><b>$ActionOMUDPSpoofDefaultTemplate</b> <templatename><br> +This setting instructs omudpspoof to use a template different from the +default template for all of its actions that do not have a template specified +explicitely. +<li><b>$ActionOMUDPSpoofSourcePortStart</b> <number><br> +Specifies the start value for circeling the source ports. Must be less than or +equal to the end value. Default is 32000. +<li><b>$ActionOMUDPSpoofSourcePortEnd</b> <number><br> +Specifies the ending value for circeling the source ports. Must be less than or +equal to the start value. Default is 42000. +</ul> +<b>Caveats/Known Bugs:</b> +<ul> +<li><b>IPv6</b> is currently not supported. If you need this capability, please let us +know via the rsyslog mailing list. +</ul> +<p><b>Sample:</b></p> +<p>The following sample forwards all syslog messages in standard form to the +remote server server.example.com. The original sender's address is used. We do not +care about the source port. This example is considered the typical use case for +omudpspoof. +</p> +<textarea rows="5" cols="80">$ModLoad omudpspoof +$ActionUDPSpoofTargetHost server.example.com +*.* :omudpspoof: +</textarea> + +<p>The following sample forwards all syslog messages in unmodified form to the +remote server server.example.com. The sender address 192.0.2.1 with fixed +source port 514 is used. +</p> +<textarea rows="8" cols="80">$ModLoad omudpspoof +$template spoofaddr,"192.0.2.1" +$template spooftemplate,"%rawmsg%" +$ActionUDPSpoofSourceNameTemplate spoofaddr +$ActionUDPSpoofTargetHost server.example.com +$ActionUDPSpoofSourcePortStart 514 +$ActionUDPSpoofSourcePortEnd 514 +*.* :omudpspoof:;spooftemplate +</textarea> +<p>The following sample is similar to the previous, but uses as many defaults as possible. +In that sample, a source port in the range 32000..42000 is used. The message is formatted +according to rsyslog's canned default forwarding format. Note that if any parameters +have been changed, the previously set defaults will be used! +</p> +<textarea rows="5" cols="80">$ModLoad omudpspoof +$template spoofaddr,"192.0.2.1" +$ActionUDPSpoofSourceNameTemplate spoofaddr +$ActionUDPSpoofTargetHost server.example.com +*.* :omudpspoof: +</textarea> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/pmlastmsg.html b/doc/pmlastmsg.html new file mode 100644 index 00000000..2abeac6a --- /dev/null +++ b/doc/pmlastmsg.html @@ -0,0 +1,62 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>parser module for "last message repeated n times" (pmlastmsg)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>parser module for "last message repeated n times" (pmlastmsg)</h1> +<p><b>Module Name: pmlastmsg</b></p> +<p><b>Module Type: parser module</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 5.5.6</p> +<p><b>Description</b>:</p> +<p>Some syslogds are known to emit severily malformed messages with content +"last message repeated n times". These messages can mess up message reception, as +they lead to wrong interpretation with the standard RFC3164 parser. Rather than +trying to fix this issue in pmrfc3164, we have created a new parser module +specifically for these messages. The reason is that some processing overhead is +involved in processing these messages (they must be recognized) and we would +not like to place this toll on every user but only on those actually in need +of the feature. Note that the performance toll is not large -- but if you expect +a very high message rate with tenthousands of messages per second, you will notice +a difference. +<p>This module should be loaded first inside <a href="messageparser.html">rsyslog's +parser chain</a>. It processes all those messages that contain a PRI, then none or +some spaces and then the exact text (case-insensitive) "last message repeated n times" +where n must be an integer. All other messages are left untouched. + +<p><b>Configuration Directives</b>:</p> +<p>There do not currently exist any configuration directives for this module. +<p><b>Examples:</b></p> +<p>This example is the typical use case, where some systems emit malformed +"repeated msg" messages. Other than that, the default RFC5424 and RFC3164 parsers +should be used. Note that when a parser is specified, the default parser chain +is removed, so we need to specify all three parsers. We use this together with the +default ruleset. +</p> +<textarea rows="15" cols="80">$ModLoad pmlastmsg # this parser is NOT a built-in module + +# note that parser are tried in the +# order they appear in rsyslog.conf, so put pmlastmsg first +$RulesetParser rsyslog.lastline +# as we have removed the default parser chain, we +# need to add the default parsers as well. +$RulesetParser rsyslog.rfc5424 +$RulesetParser rsyslog.rfc3164 + +# now come the typical rules, like... +*.* /path/to/file.log +</textarea> +<p><b>Caveats/Known Bugs:</b> +<p>currently none +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/queue_msg_state.dot b/doc/queue_msg_state.dot new file mode 100644 index 00000000..bfef2657 --- /dev/null +++ b/doc/queue_msg_state.dot @@ -0,0 +1,25 @@ +// This file is part of rsyslog. +// +// rsyslog message state in queue processing +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + rankdir=LR + + prod [label="producer" style="dotted" shape="box"] + que [label="queued"] + deq [label="dequeued"] + del [label="deleted"] + + prod -> que [label="qEnq()" style="dotted"] + que -> deq [label="qDeq()"] + deq -> del [label="qDel()"] + deq -> que [label="fatal failure\n& restart"] + + //{rank=same; del apf pdn } +} diff --git a/doc/queue_msg_state.jpeg b/doc/queue_msg_state.jpeg Binary files differnew file mode 100644 index 00000000..a215f000 --- /dev/null +++ b/doc/queue_msg_state.jpeg diff --git a/doc/queues.html b/doc/queues.html index 45ce1bd1..75b70fbf 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -336,6 +336,33 @@ in this regard - it was just not requested so far. So if you need more fine-grained control, let us know and we'll probably implement it. There are two configuration directives, both should be used together or results are unpredictable:" <i>$<object>QueueDequeueTimeBegin <hour></i>" and "<i>$<object>QueueDequeueTimeEnd <hour></i>". The hour parameter must be specified in 24-hour format (so 10pm is 22). A use case for this parameter can be found in the <a href="http://wiki.rsyslog.com/index.php/OffPeakHours">rsyslog wiki</a>. </p> +<h2>Performance</h2> +<p>The locking involved with maintaining the queue has a potentially large +performance impact. How large this is, and if it exists at all, depends much on +the configuration and actual use case. However, the queue is able to work on +so-called "batches" when dequeueing data elements. With batches, +multiple data elements are dequeued at once (with a single locking call). +The queue dequeues all available elements up to a configured upper +limit (<i><object>DequeueBatchSize <number></i>). It is important +to note that the actual upper limit is dictated by availability. The queue engine +will never wait for a batch to fill. So even if a high upper limit is configured, +batches may consist of fewer elements, even just one, if there are no more elements +waiting in the queue. +<p>Batching +can improve performance considerably. Note, however, that it affects the +order in which messages are passed to the queue worker threads, as each worker +now receive as batch of messages. Also, the larger the batch size and the higher +the maximum number of permitted worker threads, the more main memory is needed. +For a busy server, large batch sizes (around 1,000 or even more elements) may be useful. +Please note that with batching, the main memory must hold BatchSize * NumOfWorkers +objects in memory (worst-case scenario), even if running in disk-only mode. So if you +use the default 5 workers at the main message queue and set the batch size to 1,000, you need +to be prepared that the main message queue holds up to 5,000 messages in main memory +<b>in addition</b> to the configured queue size limits! +<p>The queue object's default maximum batch size +is eight, but there exists different defaults for the actual parts of +rsyslog processing that utilize queues. So you need to check these object's +defaults. <h2>Terminating Queues</h2> <p>Terminating a process sounds easy, but can be complex. Terminating a running queue is in fact the most complex operation a queue diff --git a/doc/rfc5424layers.png b/doc/rfc5424layers.png Binary files differnew file mode 100644 index 00000000..70192cc0 --- /dev/null +++ b/doc/rfc5424layers.png diff --git a/doc/rsconf1_abortonuncleanconfig.html b/doc/rsconf1_abortonuncleanconfig.html new file mode 100644 index 00000000..77526c07 --- /dev/null +++ b/doc/rsconf1_abortonuncleanconfig.html @@ -0,0 +1,37 @@ +<html> +<head> +<title>rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">rsyslog.conf configuration directive</a> + +<h2>$AboortOnUncleanConfig</h2> +<p><b>Type:</b> global configuration directive</p> +<p><b>Parameter Values:</b> boolean (on/off, yes/no)</p> +<p><b>Available since:</b> 5.3.1+</p> +<p><b>Default:</b> off</p> +<p><b>Description:</b></p> +<p>This directive permits to prevent rsyslog from running when the configuration file +is not clean. "Not Clean" means there are errors or some other annoyances that rsyslgod +reports on startup. This is a user-requested feature to have a strict startup mode. Note +that with the current code base it is not always possible to differentiate between an +real error and a warning-like condition. As such, the startup will also prevented if +warnings are present. I consider this a good thing in being "strict", but I admit +there also currently is no other way of doing it. +<p><b>Caveats:</b></p> +Note that the consequences of a failed rsyslogd startup can be much more serious than a +startup with only partial configuration. For example, log data may be lost or systems that +depend on the log server in question will not be able to send logs, what in the ultimate +result could result in a system hang on those systems. Also, the local system may hang when +the local log socket has become full and is not read. There exist many such scenarios. +As such, it is strongly recommended not to turn on this directive. + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_escape8bitcharsonreceive.html b/doc/rsconf1_escape8bitcharsonreceive.html new file mode 100644 index 00000000..408851c1 --- /dev/null +++ b/doc/rsconf1_escape8bitcharsonreceive.html @@ -0,0 +1,44 @@ +<html> +<head> +<title>rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">back</a> + +<h2>$Escape8BitCharactersOnReceive</h2> +<p><b>Type:</b> global configuration directive</p> +<p><b>Default:</b> off</p> +<p><b>Available Since:</b> 5.5.2</p> +<p><b>Description:</b></p> +<p>This directive instructs rsyslogd to replace non US-ASCII characters (those that +have the 8th bit set) during reception of the message. +This may be useful for some systems. +Please note that this escaping breaks Unicode and many other encodings. Most importantly, +it can be assumed that Asian and European characters will be rendered hardly readable by +this settings. However, it may still be useful when the logs themself are primarily +in English and only occasionally contain local script. +If this option is turned on, all control-characters are converted to a 3-digit octal number and be prefixed with the $ControlCharacterEscapePrefix character (being '#' by default). +<p><b>Warning:</b></p> +<ul> + <li>turning on this option most probably destroys non-western character sets + (like Japanese, Chinese and Korean) as well as European character sets.</li> + <li>turning on this option destroys digital signatures if such exists inside + the message</li> + <li>if turned on, the drop-cc, space-cc and escape-cc + <a href="property_replacer.html">property replacer</a> options do not work + as expected because control characters are already removed upon message + reception. If you intend to use these property replacer options, you must + turn off $Escape8BitCharactersOnReceive.</li> +</ul> +<p><b>Sample:</b></p> +<p><code><b>$Escape8BitCharactersOnReceive on</b></code></p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_rulesetcreatemainqueue.html b/doc/rsconf1_rulesetcreatemainqueue.html new file mode 100644 index 00000000..5c1e0dec --- /dev/null +++ b/doc/rsconf1_rulesetcreatemainqueue.html @@ -0,0 +1,83 @@ +<html> +<head> +<title>RulesetCreateMainQueue - rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">rsyslog.conf configuration directive</a> + +<h2>$RulesetCreateMainQueue</h2> +<p><b>Type:</b> ruleset-specific configuration directive</p> +<p><b>Parameter Values:</b> boolean (on/off, yes/no)</p> +<p><b>Available since:</b> 5.3.5+</p> +<p><b>Default:</b> off</p> +<p><b>Description:</b></p> +<p> +Rulesets may use their own "main" message queue for message submission. Specifying +this directive, <b>inside a ruleset definition</b>, turns this on. This is both a performance +enhancement and also permits different rulesets (and thus different inputs within the same +rsyslogd instance) to use different types of main message queues. +<p>The ruleset queue is created with the parameters that are specified for the main message +queue at the time the directive is given. If different queue configurations are desired, +different main message queue directives must be used in front of the $RulesetCreateMainQueue +directive. Note that this directive may only be given once per ruleset. If multiple statements +are specified, only the first is used and for the others error messages are emitted. +<p>Note that the final set of ruleset configuration directives specifies the parameters for +the default main message queue. +<p>To learn more about this feature, please be sure to read about +<a href="multi_ruleset.html">multi-ruleset support in rsyslog</a>. +<p><b>Caveats:</b></p> +The configuration statement "$RulesetCreateMainQueue off" has no effect at all. +The capability to specify this is an artifact of the current (ugly!) configuration +language. + +<p><b>Example:</b></p> +<p>This example sets up a tcp server with three listeners. Each of these +three listener is bound to a specific ruleset. As a performance optimization, +the rulesets all receive their own private queue. The result is that received messages +can be independently processed. With only a single main message queue, we would have +some lock contention between the messages. This does not happen here. Note that in this +example, we use different processing. Of course, all messages could also have been +processed in the same way ($IncludeConfig may be useful in that case!). +</p> +<textarea rows="30" cols="60">$ModLoad imtcp +# at first, this is a copy of the unmodified rsyslog.conf +#define rulesets first +$RuleSet remote10514 +$RulesetCreateMainQueue on # create ruleset-specific queue +*.* /var/log/remote10514 + +$RuleSet remote10515 +$RulesetCreateMainQueue on # create ruleset-specific queue +*.* /var/log/remote10515 + +$RuleSet remote10516 +$RulesetCreateMainQueue on # create ruleset-specific queue +mail.* /var/log/mail10516 +& ~ +# note that the discard-action will prevent this messag from +# being written to the remote10516 file - as usual... +*.* /var/log/remote10516 + +# and now define listners bound to the relevant ruleset +$InputTCPServerBindRuleset remote10514 +$InputTCPServerRun 10514 + +$InputTCPServerBindRuleset remote10515 +$InputTCPServerRun 10515 + +$InputTCPServerBindRuleset remote10516 +$InputTCPServerRun 10516 +</textarea> +<p>Note the positions of the directives. With the current config language, +position is very important. This is ugly, but unfortunately the way it currently +works. +</p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_rulesetparser.html b/doc/rsconf1_rulesetparser.html new file mode 100644 index 00000000..ef29c2a8 --- /dev/null +++ b/doc/rsconf1_rulesetparser.html @@ -0,0 +1,123 @@ +<html> +<head> +<title>RulesetParser - rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">rsyslog.conf configuration directive</a> + +<h2>$RulesetParser</h2> +<p><b>Type:</b> ruleset-specific configuration directive</p> +<p><b>Parameter Values:</b> string</p> +<p><b>Available since:</b> 5.3.4+</p> +<p><b>Default:</b> rsyslog.rfc5424 followed by rsyslog.rfc5425</p> +<p><b>Description:</b></p> +<p> +This directive permits to specify which +<a href="messageparser.html">message parsers</a> should be used for the ruleset +in question. It no ruleset is explicitely specified, the default ruleset is used. Message +parsers are contained in (loadable) parser modules with the most common cases +(RFC3164 and RFC5424) being build-in into rsyslogd. +<p>When this directive is specified the first time for a ruleset, it will not only add the +parser to the ruleset's parser chain, it will also wipe out the default parser chain. +So if you need to have +them in addition to the custom parser, you need to specify those as well. +<p>Order of directives is important. Parsers are tried one after another, in the order +they are specified inside the config. As soon as a parser is able to parse the message, +it will do so and no other parsers will be executed. If no matching parser can be found, +the message will be discarded and a warning message be issued (but only for the first +1,000 instances of this problem, to prevent message generation loops). +<p>Note that the rfc3164 parser will <b>always</b> be able to parse a message - it may +just not be the format that you like. This has two important implications: 1) always place +that parser at the END of the parser list, or the other parsers after it will never +be tried and 2) if you would like to make sure no message is lost, placing the rfc3164 +parser at the end of the parser list ensures that. +<p>Multiple parser modules are very useful if you have various devices that emit +messages that are malformed in various ways. The route to take then is +<ul> +<li>make sure you find a custom parser for that device; if there is no one, you +may consider writing one yourself (it is not that hard) or getting one written +as part of +<a href="http://www.rsyslog.com/professional-services">Adiscon's professional services +for rsyslog</a>. +<li>load your custom parsers via $ModLoad +<li>create a ruleset for each malformed format; assign the custom parser to it +<li>create a specific listening port for all devices that emit the same +malformed format +<li>bind the listener to the ruleset with the required parser +</ul> +<p>Note that it may be cumbersome to add all rules to all rulesets. To avoid this, +you can either use $Include or <a href="omruleset.html">omruleset</a> +(what probably provides the best solution). +<p>More information about rulesets in general can be found in +<a href="multi_ruleset.html">multi-ruleset support in rsyslog</a>. +<p><b>Caveats:</b></p> +<p>currently none known</p> + +<p><b>Example:</b></p> +<p>This example assumes there are two devices emiting malformed messages via UDP. +We have two custom parsers for them, named "device1.parser" and +"device2.parser". In addition to that, we have a number of other +devices sending wellformed messages, also via UDP. +<p>The solution is to listen for data from the two devices on two special +ports (10514 and 10515 in this example), create a ruleset for each and +assign the custom parsers to them. The rest of the messages are received via +port 514 using the regular parsers. Processing shall be equal for all messages. +So we simply forward the malformed messages to the regular queue once they are parsed (keep +in mind that a message is never again parsed once any parser properly processed it). +</p> +<textarea rows="40" cols="80">$ModLoad imudp +$ModLoad pmdevice1 # load parser "device1.parser" for device 1 +$ModLoad pmdevice2 # load parser "device2.parser" for device 2 + +# define ruleset for the first device sending malformed data +$Ruleset maldev1 +$RulesetCreateMainQueue on # create ruleset-specific queue +$RulesetParser "device1.parser" # note: this deactivates the default parsers +# forward all messages to default ruleset: +$ActionOmrulesetRulesetName RSYSLOG_DefaultRuleset +*.* :omruleset: + +# define ruleset for the second device sending malformed data +$Ruleset maldev2 +$RulesetCreateMainQueue on # create ruleset-specific queue +$RulesetParser "device2.parser" # note: this deactivates the default parsers +# forward all messages to default ruleset: +$ActionOmrulesetRulesetName RSYSLOG_DefaultRuleset +*.* :omruleset: + +# switch back to default ruleset +$Ruleset RSYSLOG_DefaultRuleset +*.* /path/to/file +auth.info @authlogger.example.net +# whatever else you usually do... + + +# now define the inputs and bind them to the rulesets +# first the default listener (utilizing the default ruleset) +$UDPServerRun 514 + +# now the one with the parser for device type 1: +$InputUDPServerBindRuleset maldev1 +$UDPServerRun 10514 + +# and finally the one for device type 2: +$InputUDPServerBindRuleset maldev2 +$UDPServerRun 10515 +</textarea> + +<p>For an example of how multiple parser can be chained (and an actual use case), please see +the example section on the <a href="pmlastmsg.html">pmlastmsg</a> parser +module. +<p>Note the positions of the directives. With the current config language, +<b>sequence of statements is very important</b>. This is ugly, but unfortunately +the way it currently works. +</p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 8c1cc9a7..b71fc761 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -17,6 +17,8 @@ appear as implementation progresses. many parameter settings modify queue parameters. If in doubt, use the default, it is usually well-chosen and applicable in most cases.</p> <ul> +<li><a href="rsconf1_abortonuncleanconfig.html">$AbortOnUncleanConfig</a> - abort startup if there is +any issue with the config file</li> <li><a href="rsconf1_actionexeconlywhenpreviousissuspended.html">$ActionExecOnlyWhenPreviousIsSuspended</a></li> <li>$ActionName <a_single_word> - used primarily for documentation, e.g. when generating a configuration graph. Available sice 4.3.1. @@ -58,6 +60,7 @@ default template for UDP and plain TCP forwarding action</li> <li>$ActionGSSForwardDefaultTemplate [templateName] - sets a new default template for GSS-API forwarding action</li> <li>$ActionQueueCheckpointInterval <number></li> +<li>$ActionQueueDequeueBatchSize <number> [default 16]</li> <li>$ActionQueueDequeueSlowdown <number> [number is timeout in <i> micro</i>seconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]</li> @@ -109,6 +112,14 @@ that it should be not be much more often than once per second).</li> <li><b>$ActionSendUDPRebindInterval</b> nbr</a>- [available since 4.3.2] - instructs the UDP send action to rebind the send socket every nbr of messages sent. Zero, the default, means that no rebind is done. This directive is useful for use with load-balancers.</li> +<li><b>$ActionWriteAllMarkMessages</b> [on/<b>off</b>]- [available since 5.1.5] - normally, mark messages +are written to actions only if the action was not recently executed (by default, recently means within the +past 20 minutes). If this setting is switched to "on", mark messages are always sent to actions, +no matter how recently they have been executed. In this mode, mark messages can be used as a kind of +heartbeat. Note that this option auto-resets to "off", so if you intend to use it with multiple +actions, it must be specified in front off <b>all</b> selector lines that should provide this +functionality. +</li> <li><a href="rsconf1_allowedsender.html">$AllowedSender</a></li> <li><a href="rsconf1_controlcharacterescapeprefix.html">$ControlCharacterEscapePrefix</a></li> <li><a href="rsconf1_debugprintcfsyslinehandlerlist.html">$DebugPrintCFSyslineHandlerList</a></li> @@ -129,6 +140,7 @@ our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a <li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li> <li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li> <li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li> +<li><a href="rsconf1_escape8bitcharsonreceive.html">$Escape8BitCharactersOnReceive</a></li> <li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> <li><b>$EscapeControlCharactersOnReceive</b> [<b>on</b>|off] - escape USASCII HT character</li> <li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> @@ -154,6 +166,8 @@ something along the lines of "/etc/init.d/rsyslog restart". rsyslog emits message on startup and shutdown as well as when it is HUPed. This information might be needed by some log analyzers. If set to off, no such status messages are logged, what may be useful for other scenarios. +[available since 4.7.0 and 5.3.0] +<li><b>$MainMsgQueueDequeueBatchSize</b> <number> [default 32]</li> <li>$MainMsgQueueDequeueSlowdown <number> [number is timeout in <i> micro</i>seconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]</li> @@ -245,10 +259,14 @@ large enough for the whole message. (Introduced with 4.1.5). Once set, it affect <li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li> <li><b>$Ruleset</b> <i>name</i> - starts a new ruleset or switches back to one already defined. All following actions belong to that new rule set. -the <i>name</i> does not yet exist, it is created. To swith back to rsyslog's +the <i>name</i> does not yet exist, it is created. To switch back to rsyslog's default ruleset, specify "RSYSLOG_DefaultRuleset") as the name. All following actions belong to that new rule set. It is advised to also read our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a>.</li> +<li><b><a href="rsconf1_rulesetcreatemainqueue.html">$RulesetCreateMainQueue</a></b> on - creates +a ruleset-specific main queue. +<li><b><a href="rsconf1_rulesetparser.html">$RulesetParser</a></b> - enables to set +a specific (list of) message parsers to be used with the ruleset. <li><b>$OptimizeForUniprocessor</b> [on/<b>off</b>] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common. [available since 4.1.0]</li> @@ -256,19 +274,6 @@ default may change as uniprocessor systems become less common. [available since to sysklogd), the domain part from a name that is within the same domain as the receiving system is stripped. If set to on, full names are always used.</li> <li>$WorkDirectory <name> (directory for spool and other work files)</li> -<li>$UDPServerAddress <IP> (imudp) -- local IP -address (or name) the UDP listens should bind to</li> -<li>$UDPServerRun <port> (imudp) -- former --r<port> option, default 514, start UDP server on this -port, "*" means all addresses</li> -<li>$UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance -optimization. Getting the system time is very costly. With this setting, imudp can -be instructed to obtain the precise time only once every n-times. This logic is -only activated if messages come in at a very fast rate, so doing less frequent -time calls should usually be acceptable. The default value is two, because we have -seen that even without optimization the kernel often returns twice the identical time. -You can set this value as high as you like, but do so at your own risk. The higher -the value, the less precise the timestamp. <li><a href="droppriv.html">$PrivDropToGroup</a></li> <li><a href="droppriv.html">$PrivDropToGroupID</a></li> <li><a href="droppriv.html">$PrivDropToUser</a></li> diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index b2830535..2a64461d 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -19,8 +19,17 @@ modules solve your need, you may consider writing one or have one written for you by <a href="http://www.rsyslog.com/professional-services">Adiscon's professional services for rsyslog</a> </b>(this often is a very cost-effective and efficient way of getting what you need). +<p>There exist different classes of loadable modules: +<ul> +<li><a href="rsyslog_conf_modules.html#im">Input Modules</a> +<li><a href="rsyslog_conf_modules.html#om">Output Modules</a> +<li><a href="rsyslog_conf_modules.html#pm">Parser Modules</a> +<li><a href="rsyslog_conf_modules.html#mm">Message Modification Modules</a> +<li><a href="rsyslog_conf_modules.html#sm">String Generator Modules</a> +<li><a href="rsyslog_conf_modules.html#lm">Library Modules</a> +</ul> -<h2>Input Modules</h2> +<a name"im"></a><h2>Input Modules</h2> <p>Input modules are used to gather messages from various sources. They interface to message generators. <ul> @@ -37,13 +46,14 @@ to message generators. <li><a href="im3195.html">im3195</a> - accepts syslog messages via RFC 3195</li> </ul> -<h2>Output Modules</h2> +<a name"om"></a><h2>Output Modules</h2> <p>Output modules process messages. With them, message formats can be transformed and messages be transmitted to various different targets. <ul> <li><a href="omsnmp.html">omsnmp</a> - SNMP trap output module</li> <li><a href="omstdout.html">omtdout</a> - stdout output module (mainly a test tool)</li> <li><a href="omrelp.html">omrelp</a> - RELP output module</li> +<li><a href="omruleset.html">omruleset</a> - forward message to another ruleset</li> <li>omgssapi - output module for GSS-enabled syslog</li> <li><a href="ommysql.html">ommysql</a> - output module for MySQL</li> <li>ompgsql - output module for PostgreSQL</li> @@ -53,21 +63,105 @@ SQLLite, Ingres, Oracle, mSQL)</li> <li><a href="ommail.html">ommail</a> - permits rsyslog to alert folks by mail if something important happens</li> <li><a href="omoracle.html">omoracle</a> - output module for Oracle (native OCI interface)</li> +<li><a href="omudpspoof.html">omudpspoof</a> - output module sending UDP syslog messages with a spoofed address</li> <li><a href="omuxsock.html">omuxsock</a> - output module Unix domain sockets</li> </ul> -<h2>Library Modules</h2> +<a name="pm"></a><h2>Parser Modules</h2> +<p>Parser modules are used to parse message content, once the message has been +received. They can be used to process custom message formats or invalidly formatted +messages. For details, please see the <a href="messageparser.html">rsyslog +message parser documentation</a>. +<p>The current modules are currently provided as part of rsyslog: +<ul> +<li>pmrfc5424[builtin] - rsyslog.rfc5424 - +parses RFC5424-formatted messages (the new syslog standard) +<li>pmrfc3164[builtin] - rsyslog.rfc3164 - +the traditional/legacy syslog parser +<li>pmrfc3164sd - rsyslog.rfc3164sd - +a contributed module supporting RFC5424 structured data inside +RFC3164 messages (not supported by the rsyslog team) +<li><a href="pmlastmsg.html">pmlastmsg</a> - rsyslog.lastmsg - +a parser module that handles the typically malformed "last messages +repated n times" messages emitted by some syslogds. +</ul> + +<a name="mm"></a><h2>Message Modification Modules</h2> +<p>Message modification modules are used to change the content of messages being processed. +They can be implemented using either the output module or the parser module interface. +From the rsyslog core's point of view, they actually are output or parser modules, it is their +implementation that makes them special. +<p>Currently, there do not exist any such modules, but could be written with +the methods the engine provides. They could be used, for example, to: +<ul> +<li>anonymize message content +<li>add dynamically computed content to message (fields) +</ul> + +<a name="lm"></a><h2>String Generator Modules</h2> +<p>String generator modules are used, as the name implies, to generate strings based +on the message content. They are currently tightly coupled with the template system. +Their primary use is to speed up template processing by providing a native C +interface to template generation. These modules exist since 5.5.6. To get an idea +of the potential speedup, the default file format, when generated by a string generator, +provides a roughly 5% speedup. For more complex strings, especially those that include +multiple regular expressions, the speedup may be considerably higher. +<p>String generator modules are written to a quite simple interface. However, a word of +caution is due: they access the rsyslog message object via a low-level interface. +That interface is not guaranteed yet to stay stable. So it may be necessary to +modify string generator modules if the interface changes. Obviously, we will not do that +without good reason, but it may happen. +<p>Rsyslog comes with a set of core, build-in string generators, which are used +to provide those default templates that we consider to be time-critical: +<ul> +<li>smfile - the default rsyslog file format +<li>smfwd - the default rsyslog (network) forwarding format +<li>smtradfile - the traditional syslog file format +<li>smfwd - the traditional syslog (network) forwarding format +</ul> +<p>Note that when you replace these defaults be some custom strings, you will +loose some performance (around 5%). For typical systems, this is not really relevant. +But for a high-performance systems, it may be very relevant. To solve that issue, create +a new string generator module for your custom format, starting out from one of the +default generators provided. If you can not do this yourself, you may want to +contact <a href="mailto:info%40adiscon.com">Adiscon</a> as we offer custom development +of string generators at a very low price. +<p>Note that string generator modules can be dynamically loaded. However, the default +ones provided are so important that they are build right into the executable. But this +does not need to be done that way (and it is straightforward to do it dynamic). + + +<a name="lm"></a><h2>Library Modules</h2> <p>Library modules provide dynamically loadable functionality for parts of rsyslog, most often for other loadable modules. They can not be user-configured and are loaded automatically by some components. They are just mentioned so that error messages that point to library moduls can be understood. No module list is provided. +<h2>Where are the modules integrated into the Message Flow?</h2> +<p>Depending on their module type, modules may access and/or modify messages at +various stages during rsyslog's processing. Note that only the "core type" (e.g. input, +output) but not any type derived from it (message modification module) specifies when +a module is called. +<p>The simplified workflow is as follows: +<p align="center"> +<img src="module_workflow.png" alt"rsyslog: loadable modules and message flow"> +<p>As can be seen, messages are received by input modules, then passed to one or many +parser modules, which generate the in-memory representation of the message and may +also modify the message itself. The, the internal representation is passed to +output modules, which may output a message and (with the interfaces newly introduced +in v5) may also modify messageo object content. +<p>String generator modules are not included inside this picture, because they are +not a required part of the workflow. If used, they operate "in front of" the +output modules, because they are called during template generation. +<p>Note that the actual flow is much more complex and depends a lot on queue and +filter settings. This graphic above is a high-level message flow diagram. + <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body> diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index baa4ce29..23a02049 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -16,6 +16,35 @@ compatible with the stock syslogd formats are hardcoded into rsyslogd. So if no template is specified, we use one of these hardcoded templates. Search for "template_" in syslogd.c and you will find the hardcoded ones.</p> +<p>Starting with 5.5.6, there are actually two differnt types of template: +<ul> +<li>string based +<li>string-generator module based +</ul> +<p><a href="rsyslog_conf_modules.html#sm">String-generator module</a> based templates +have been introduced in 5.5.6. They permit a string generator, actually a C "program", +the generate a format. Obviously, it is more work required to code such a generator, +but the reward is speed improvement. If you do not need the ultimate throughput, you +can forget about string generators (so most people never need to know what they are). +You may just be interested in learning that for the most important default formats, +rsyslog already contains highly optimized string generators and these are called +without any need to configure anything. But if you have written (or purchased) a +string generator module, you need to know how to call it. Each such module has a name, +which you need to know (look it up in the module doc or ask the developer). Let's assume +that "mystrgen" is the module name. Then you can define a template for that strgen +in the following way: +<blockquote><code>$template MyTemplateName,=mystrgen</code></blockquote> +(Of course, you must have first loaded the module via $ModLoad). +<p>The important part is the equal sign: it tells the rsyslog config parser that +no string follows but a strgen module name. +<p>There are no additional parameters but the module name supported. This is because +there is no way to customize anything inside such a "template" other than by +modifying the code of the string generator. + +<p>So for most use cases, string-generator module based templates are <b>not</b> +the route to take. Usually, us use <b>string based templates</b> instead. +This is what the rest of the documentation now talks about. + <p>A template consists of a template directive, a name, the actual template text and optional options. A sample is:</p> <blockquote><code>$template MyTemplateName,"\7Text @@ -136,6 +165,23 @@ out, but this may happen.</li> is meant to be written to a log file. Do <b>not</b> use for production or remote forwarding.</li> </ul> +<h3>String-based Template Samples</h3> +<p>This section provides some sample of what the default formats would +look as a text-based template. Hopefully, their description is self-explanatory. +Note that each $Template statement is on a <b>single</b> line, but probably broken +accross several lines for display purposes by your browsers. Lines are separated by +empty lines. +<p><code> +$template FileFormat,"%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" +<br><br> +$template TraditionalFileFormat,"%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" +<br><br> +$template ForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" +<br><br> +$template TraditionalForwardFormat,"<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" +<br><br> +$template StdSQLFormat,"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-mysql%', '%timegenerated:::date-mysql%', %iut%, '%syslogtag%')",SQL +</code></p> <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] diff --git a/doc/rsyslog_ng_comparison.html b/doc/rsyslog_ng_comparison.html index 8e121a8d..7d12a4a7 100644 --- a/doc/rsyslog_ng_comparison.html +++ b/doc/rsyslog_ng_comparison.html @@ -5,6 +5,10 @@ <h1>rsyslog vs. syslog-ng</h1> <p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> (2008-05-06)</i></small></p> +<p><i>Warning</i>: this comparison is a little outdated, take it with a grain +of salt and be sure to check the links at the bottom (both syslog-ng as well as +rsyslog features are missing, but our priority is on creating great software not +continously updating this comparison ;)). <p>We have often been asked about a comparison sheet between rsyslog and syslog-ng. Unfortunately, I do not know much about syslog-ng, I did not even use it once. Also, there seems to be no @@ -81,9 +85,10 @@ optional input</td> </tr> <tr> <td valign="top">Windows Event Log</td> -<td valign="top">via <a href="http://www.eventreporter.com">EventReporter</a> +<td valign="top">via a Windows event logging software such as +<a href="http://www.eventreporter.com">EventReporter</a> or <a href="http://www.mwagent.com">MonitorWare Agent</a> -(both commercial software)</td> +(both commercial software, both fund rsyslog development)</td> <td valign="top">via separate Windows agent, paid edition only</td> </tr> diff --git a/doc/rsyslog_queue_pointers.jpeg b/doc/rsyslog_queue_pointers.jpeg Binary files differnew file mode 100644 index 00000000..809dd446 --- /dev/null +++ b/doc/rsyslog_queue_pointers.jpeg diff --git a/doc/rsyslog_queue_pointers2.jpeg b/doc/rsyslog_queue_pointers2.jpeg Binary files differnew file mode 100644 index 00000000..2ad60113 --- /dev/null +++ b/doc/rsyslog_queue_pointers2.jpeg diff --git a/doc/rsyslog_secure_tls.html b/doc/rsyslog_secure_tls.html index be2811f4..b15e5a4e 100644 --- a/doc/rsyslog_secure_tls.html +++ b/doc/rsyslog_secure_tls.html @@ -51,7 +51,7 @@ google_ad_height = 125; src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> </script> </span> -I private keys have become known to third parties, the system does not provide +If private keys have become known to third parties, the system does not provide any security at all. Also, our solution bases on X.509 certificates and a (very limited) chain of trust. We have one instance (the CA) that issues all machine certificates. The machine certificate indentifies a particular machine. hile in diff --git a/doc/src/classes.dia b/doc/src/classes.dia Binary files differindex 70e91566..8cfcbd0c 100644 --- a/doc/src/classes.dia +++ b/doc/src/classes.dia diff --git a/doc/src/module_workflow.dia b/doc/src/module_workflow.dia Binary files differnew file mode 100644 index 00000000..178571f4 --- /dev/null +++ b/doc/src/module_workflow.dia diff --git a/doc/src/rfc5424layers.dia b/doc/src/rfc5424layers.dia Binary files differnew file mode 100644 index 00000000..300b7796 --- /dev/null +++ b/doc/src/rfc5424layers.dia diff --git a/doc/src/rsyslog_queue_pointers.dia b/doc/src/rsyslog_queue_pointers.dia Binary files differnew file mode 100644 index 00000000..2ad4cacb --- /dev/null +++ b/doc/src/rsyslog_queue_pointers.dia diff --git a/doc/src/rsyslog_queue_pointers2.dia b/doc/src/rsyslog_queue_pointers2.dia Binary files differnew file mode 100644 index 00000000..6a35c664 --- /dev/null +++ b/doc/src/rsyslog_queue_pointers2.dia diff --git a/doc/src/tls.dia b/doc/src/tls.dia Binary files differindex 77e5d185..d7c9811d 100644 --- a/doc/src/tls.dia +++ b/doc/src/tls.dia diff --git a/doc/status.html b/doc/status.html deleted file mode 100644 index 4e8f1a5f..00000000 --- a/doc/status.html +++ /dev/null @@ -1,54 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html><head><title>rsyslog status page</title></head> -<body> -<h2>rsyslog status page</h2> -<p>This page reflects the status as of 2009-05-25.</p> -<h2>Current Releases</h2> - -<p><b>development:</b> 4.3.1 [2009-05-25] - -<a href="http://www.rsyslog.com/Article372.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-159.phtml">download</a> - -<br><b>beta:</b> 3.21.11 [2009-04-03] - -<a href="http://www.rsyslog.com/Article358.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-152.phtml">download</a></p> - -<p><b>v3 stable:</b> 3.22.0 [2009-04-21] - <a href="http://www.rsyslog.com/Article368.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-157.phtml">download</a> - -<br><b>v2 stable:</b> 2.0.7 [2009-04-14] - <a href="http://www.rsyslog.com/Article362.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-154.phtml">download</a> -<br>v0 and v1 are deprecated and no longer supported. If you absolutely do not like to -upgrade, you may consider purchasing a -<a href="professional_support.html">commercial rsyslog support package</a>. Just let us point -out that it is really not a good idea to still run a v0 version. - -<p><a href="v3compatibility.html">If you updgrade from version 2, be sure to read the rsyslog v3 -compatibility document.</a> There are no additional compatibility concerns at this time for -upgrading from v3 to v4. If some occur, we will post an additional compatiblity document.</p> -<p>(<a href="version_naming.html">How are versions named?</a>)</p> - -<h2>Platforms</h2> -<p>Thankfully, a number of folks have begin to build packages and -help port rsyslog to other platforms. As such, -<a href="http://wiki.rsyslog.com/index.php/Platforms">the -platform list is now maintained inside the rsyslog wiki</a>. -Platform maintainers perhaps have posted extra information there. If -you do platform-specific work, feel free to add information to the wiki.</p> -<p>Rsyslog is the default syslogd in Fedora 8 and above.</p> -<h2>Additional information</h2> -<p><b>Currently supported features are listed on the <a href="features.html">rsyslog features page</a>.</b></p> -<ul> -<li>The rsyslog home page is <a href="http://www.rsyslog.com">www.rsyslog.com</a>.</li> -<li>Mailing list info can be found at <a href="http://lists.adiscon.net/mailman/listinfo/rsyslog">http://lists.adiscon.com/rsyslog</a>.</li> -<li>The change log can be found at <a href="http://www.rsyslog.com/Topic4.phtml"> -http://www.rsyslog.com/Topic4.phtml</a>. </li> -<li>Online documentation is available at <a href="http://www.rsyslog.com/doc">http://www.rsyslog.com/doc</a>.</li> -<li>You may also find <a href="http://rgerhards.blogspot.com/">Rainer's blog</a> -an interesting read.</li> -</ul> -<p>The project was initiated in 2003 and seriouosly begun in 2004 by -<a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> -and is currently being maintained by him. See the <a href="history.html">history page</a> for more -background information.</p> -</body></html> diff --git a/doc/syslog_parsing.html b/doc/syslog_parsing.html index 57da6657..1ccec6f1 100644 --- a/doc/syslog_parsing.html +++ b/doc/syslog_parsing.html @@ -176,6 +176,19 @@ $template, MalfromedMsgFormater,"%timegenerated% %fromhost% %rawmsg:::drop-last- <p>This will make your log much nicer, but not look perfect. Experiment a bit with the available properties and replacer extraction options to fine-tune it to your needs. +<h2>The Ultimate Solution...</h2> +<p>Is available with rsyslog 5.3.4 and above. Here, we can define so-called custom +parsers. These are plugin modules, written in C and adapted to a specific message format +need. The big plus of custom parsers is that they offer excellent performance and unlimited +possibilities - far better than any work-around could do. Custom parsers can be +<a href="rsconf1_rulesetparser.html">bound to specific rule sets</a> +(and thus listening) ports with relative ease. The only con is that they must be written. +However, if you are lucky, a parser for your device may already exist. If not, you can +opt to write it yourself, what is not too hard if you know some C. Alternatively, +Adiscon can program one for you as part of the +<a href="http://www.rsyslog.com/professional-services">rsyslog professional services offering</a>. +In any case, you should seriously consider custom parsers as an alternative if you can not +reconfigure your device to send decent message format. <h2>Wrap-Up</h2> <p>Syslog message format is not sufficiently standardized. There exists a weak "standard" format, which is used by a good number of implementations. However, there @@ -183,14 +196,15 @@ exist many others, including mainstream vendor implementations, which have a (sometimes horribly) different format. Rsyslog tries to deal with anomalies but can not guess right in all instances. If possible, the sender should be configured to submit well-formed messages. If that is not possible, you can work around these -issues with rsyslog's property replacer and template system. +issues with rsyslog's property replacer and template system. Or you can use a suitable +message parser or write one for your needs. <p>I hope this is a useful guide. You may also have a look at the <a href="troubleshoot.html">rsyslog troubleshooting guide</a> for further help and places where to ask questions. <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body></html> diff --git a/doc/tls_cert_server.html b/doc/tls_cert_server.html index 9c68db5d..9c024bc9 100644 --- a/doc/tls_cert_server.html +++ b/doc/tls_cert_server.html @@ -37,6 +37,15 @@ src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> </script> </span> <p><center><img src="tls_cert_100.jpg"></center> +<p><i><font color="red"><b>Important:</b> Keep in mind that the order of configuration directives +is very important in rsyslog. As such, the samples given below do only work if the given +order is preserved.</font> Re-ordering the directives can break configurations and has broken them +in practice. If you intend to re-order them, please be sure that you fully understand how +the configuration language works and, most importantly, which statements form a block together. +Please also note that we understand the the current configuration file format is +ugly. However, there has been more important work in the way of enhancing it. If you would like +to contribute some time to improve the config file language, please let us know. Any help +is appreciated (be it doc or coding work!).</i> <p>Steps to do: <ul> <li>make sure you have a functional CA (<a href="tls_cert_ca.html">Setting up the CA</a>) diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index a8855fd4..0f0c7fca 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -102,13 +102,63 @@ comes without any guarantees, include no guarantee on confidentiality [aka "we don't want to be sued for work were are not even paid for ;)]. <b>So if you submit debug logs, do so at your sole risk</b>. By submitting them, you accept this policy. +<p><b>Segmentation Faults</b> +<p>Rsyslog has a very rapid development process, complex capabilities and now gradually gets +more and more exposure. While we are happy about this, it also has some bad effects: some +deployment scenarios have probably never been tested and it may be impossible to test +them for the development team because of resources needed. So while we try to avoid this, +you may see a serious problem during deployments in demanding, non-standard, environments +(hopefully not with a stable version, but chances are good you'll run into troubles with +the development versions). +<p>Active support from the user base is very important to help us track down those things. +Most often, serious problems are the result of some memory misadressing. During development, +we routinely use valgrind, a very well and capable memory debugger. This helps us to create +pretty clean code. But valgrind can not detect everything, most importantly not code pathes +that are never executed. So of most use for us is information about aborts and abort locations. +<p>Unforutnately, faults rooted in adressing errors typically show up only later, so the +actual abort location is in an unrelated spot. To help track down the original spot, +<a href="http://www.gnu.org/software/hello/manual/libc/Heap-Consistency-Checking.html">libc +later than 5.4.23 offers support</a> for finding, and possible temporary relief from it, +by means of the MALLOC_CHECK_ environment variable. Setting it to 2 is a useful troubleshooting +aid for us. It will make the program abort as soon as the check routines detect anything +suspicious (unfortunately, this may still not be the root cause, but hopefully closer to it). +Setting it to 0 may even make some problems disappear (but it will NOT fix them!). +With functionality comes cost, and so exporting MALLOC_CHECK_ without need comes at +a performance penalty. However, we strongly recommend adding this instrumentation to your +test environment should you see any serious problems. Chances are good it will help us +interpret a dump better, and thus be able to quicker craft a fix. +<p>In order to get useful information, we need some backtrace of the abort. First, you need +to make sure that a core file is created. Under Fedora, for example, that means you need +to have an "ulimit -c unlimited" in place. +<p>Now let's assume you got a core file (e.g. in /core.1234). So what to do next? Sending a +core file to us is most often pointless - we need to have the exact same system configuration in +order to interpret it correctly. Obviously, chances are extremely slim for this to be. So we would +appreciate if you could extract the most important information. This is done as follows: +<ul> +<li>$gdb /path/to/rsyslogd +<li>$info thread +<li>you'll see a number of threads (in the range 0 to n with n being the highest number). For + <b>each</b> of them, do the following (let's assume that i is the thread number): + <ul> + <li>$ thread i (e.g. thread 0, thread 1, ...) + <li>$bt + </ul> +<li>then you can quit gdb with "$q" +</ul> +<p>Then please send all information that gdb spit out to the development team. It is best to first +ask on the forum or mailing list on how to do that. The developers will keep in contact with you +and, I fear, will probably ask for other things as well ;) +<p>Note that we strive for highest reliability of the engine even in unusual deployment scenarios. +Unfortunately, this is hard to achieve, especially with limited resources. So we are depending on +cooperation from users. This is your chance to make a big contribution to the project without the +need to program or do anything else except get a problem solved ;) <p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/v5compatibility.html b/doc/v5compatibility.html new file mode 100644 index 00000000..6d60062f --- /dev/null +++ b/doc/v5compatibility.html @@ -0,0 +1,30 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Compatibility notes for rsyslog v5</title> +</head> +<body> +<h1>Compatibility Notes for rsyslog v5</h1> +<p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> +(2009-07-15)</i></small></p> +<p>The changes introduced in rsyslog v5 are numerous, but not very intrusive. +This document describes things to keep in mind when moving from v4 to v5. It +does not list enhancements nor does it talk about compatibility concerns introduced +by earlier versions (for this, see their respective compatibility documents). +<h2>HUP processing</h2> +<p>The $HUPisRestart directive is supported by some early v5 versions, but has been removed +in 5.1.3 and above. That means that restart-type HUP processing is no longer +available. This processing was redundant and had a lot a drawbacks. +For details, please see the +<a href="v4compatibility.html">rsyslog v4 compatibility notes</a> which elaborate +on the reasons and the (few) things you may need to change. +<h2>Queue Worker Thread Shutdown</h2> +<p>Previous rsyslog versions had the capability to "run" on zero queue worker +if no work was required. This was done to save a very limited number of resources. However, +it came at the price of great complexity. In v5, we have decided to let a minium of one +worker run all the time. The additional resource consumption is probably not noticable at +all, however, this enabled us to do some important code cleanups, resulting in faster +and more reliable code (complex code is hard to maintain and error-prone). From the +regular user's point of view, this change should be barely noticable. I am including the +note for expert users, who will notice it in rsyslog debug output and other analysis tools. +So it is no error if each queue in non-direct mode now always runs at least one worker +thread. +</body></html> @@ -51,6 +51,7 @@ #include "obj.h" #include "errmsg.h" #include "gss-misc.h" +#include "debug.h" #include "glbl.h" #include "unlimited_select.h" @@ -190,7 +191,7 @@ static int recv_token(int s, gss_buffer_t tok) | lenbuf[3]); tok->length = ntohl(len); - tok->value = (char *) malloc(tok->length ? tok->length : 1); + tok->value = (char *) MALLOC(tok->length ? tok->length : 1); if (tok->length && tok->value == NULL) { errmsg.LogError(0, NO_ERRCODE, "Out of memory allocating token data\n"); return -1; diff --git a/java/Makefile.am b/java/Makefile.am new file mode 100644 index 00000000..67f5eb43 --- /dev/null +++ b/java/Makefile.am @@ -0,0 +1,35 @@ +# very rough support for compiling the java components of rsyslog +# Some usage notes: you need to use the Sun JDK compiler (jdk-devel) +# with this. At least it didn't work for me with the eclipse compiler. +# There is no real installation support. If you intend to run a program, +# change to the ./java subdirectory and issue +# java -cp . <class> +# e.g.: java -cp . com.rsyslog.gui.diaggui.DiagGUI +# or any equivalent command. +# +# I am very glad to hear suggestions about how to improve this part +# of the build system. -- rgerhards, 2009-08-27 + +javadir = $(top_builddir)/java +JAVAROOT = $(javadir) +# I don't know why CLASSPATH_ENV works this way, but at least it works... +CLASSPATH_ENV = CLASSPATH=$(javadir):$$CLASSPATH + +JAVA_SOURCE_FILES = \ + com/rsyslog/lib/DiagSess.java \ + com/rsyslog/lib/SyslogMessage.java \ + com/rsyslog/lib/SyslogMsgConsumer.java \ + com/rsyslog/lib/SyslogTrafficGenerator.java \ + com/rsyslog/lib/SyslogSender.java \ + com/rsyslog/lib/UDPSyslogSender.java \ + com/rsyslog/diag/DiagTalker.java \ + com/rsyslog/gui/simpServ/simpServ.java \ + com/rsyslog/gui/simpServ/simpServConsumer.java \ + com/rsyslog/gui/msggen/MsgGen.java \ + com/rsyslog/gui/diaggui/Counters.java \ + com/rsyslog/gui/diaggui/DiagGUI.java + + +java_JAVA = $(JAVA_SOURCE_FILES) + +dist_java = $(JAVA_SOURCE_FILES) diff --git a/java/com/rsyslog/diag/DiagTalker.java b/java/com/rsyslog/diag/DiagTalker.java new file mode 100644 index 00000000..c4e77e95 --- /dev/null +++ b/java/com/rsyslog/diag/DiagTalker.java @@ -0,0 +1,70 @@ +/* A yet very simple tool to talk to imdiag. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.diag; +import java.io.*; +import java.net.*; + +public class DiagTalker { + public static void main(String[] args) throws IOException { + + Socket diagSocket = null; + PrintWriter out = null; + BufferedReader in = null; + final String host = "127.0.0.1"; + final int port = 13500; + + try { + diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(0); /* wait for lenghty operations */ + out = new PrintWriter(diagSocket.getOutputStream(), true); + in = new BufferedReader(new InputStreamReader( + diagSocket.getInputStream())); + } catch (UnknownHostException e) { + System.err.println("can not resolve " + host + "!"); + System.exit(1); + } catch (IOException e) { + System.err.println("Couldn't get I/O for " + + "the connection to: " + host + "."); + System.exit(1); + } + + BufferedReader stdIn = new BufferedReader( + new InputStreamReader(System.in)); + String userInput; + + try { + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + } catch (SocketException e) { + System.err.println("We had a socket exception and consider this to be OK: " + + e.getMessage()); + } + + out.close(); + in.close(); + stdIn.close(); + diagSocket.close(); + } +} + diff --git a/java/com/rsyslog/gui/diaggui/Counters.java b/java/com/rsyslog/gui/diaggui/Counters.java new file mode 100644 index 00000000..363bff43 --- /dev/null +++ b/java/com/rsyslog/gui/diaggui/Counters.java @@ -0,0 +1,138 @@ +/* Display some basic rsyslogd counter variables. + * + * Please note that this program requires imdiag to be loaded inside rsyslogd. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.diaggui; +import java.awt.*; +import java.awt.event.*; +import java.io.*; +import java.util.*; + +import com.rsyslog.lib.DiagSess; + +public class Counters extends Frame { + + private TextField MainQItems; + private TextField RefreshInterval; + private Checkbox AutoRefresh; + private DiagSess diagSess; + private Timer timer; + + private void createDiagSess() { + try { + diagSess = new DiagSess("127.0.0.1", 13500); // TODO: values from GUI + diagSess.connect(); + } + catch(IOException e) { + System.out.println("Exception trying to open diag session:\n" + e.toString()); + } + } + + private void createGUI() { + MainQItems = new TextField(); + MainQItems.setColumns(8); + Panel pCenter = new Panel(); + pCenter.setLayout(new FlowLayout()); + pCenter.add(new Label("MainQ Items:")); + pCenter.add(MainQItems); + + RefreshInterval = new TextField(); + RefreshInterval.setColumns(5); + RefreshInterval.setText("100"); + AutoRefresh = new Checkbox("Auto Refresh", false); + AutoRefresh.addItemListener(new ItemListener() { + public void itemStateChanged(ItemEvent e) { + setAutoRefresh(); + } + + }); + Button b = new Button("Refresh now"); + b.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + refreshCounters(); + } + }); + Panel pSouth = new Panel(); + pSouth.setLayout(new FlowLayout()); + pSouth.add(AutoRefresh); + pSouth.add(new Label("Interval (ms):")); + pSouth.add(RefreshInterval); + pSouth.add(b); + + pack(); + setTitle("rsyslogd Counters"); + setLayout(new BorderLayout()); + add(pCenter, BorderLayout.CENTER); + add(pSouth, BorderLayout.SOUTH); + setSize(400,500); + } + + public Counters() { + addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e){ + Counters.this.dispose(); + } + }); + createGUI(); + createDiagSess(); + setAutoRefresh(); + setVisible(true); + } + + + private void startTimer() { + timer = new Timer(); + timer.scheduleAtFixedRate(new TimerTask() { + public void run() { + refreshCounters(); + } + }, 0, 100); + } + + private void stopTimer() { + if(timer != null) { + timer.cancel(); + timer = null; + } + } + + /** set auto-refresh mode. It is either turned on or off, depending on the + * status of the relevant check box. */ + private void setAutoRefresh() { + if(AutoRefresh.getState() == true) { + startTimer(); + } else { + stopTimer(); + } + } + + /** refresh counter display from rsyslogd data. Does a network round-trip. */ + private void refreshCounters() { + try { + String res = diagSess.request("getmainmsgqueuesize"); + MainQItems.setText(res); + } + catch(IOException e) { + System.out.println("Exception during request:\n" + e.toString()); + } + } +} diff --git a/java/com/rsyslog/gui/diaggui/DiagGUI.java b/java/com/rsyslog/gui/diaggui/DiagGUI.java new file mode 100644 index 00000000..1a03299c --- /dev/null +++ b/java/com/rsyslog/gui/diaggui/DiagGUI.java @@ -0,0 +1,77 @@ +/* A yet very simple diagnostic GUI for rsyslog. + * + * Please note that this program requires imdiag to be loaded inside rsyslogd. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.diaggui; +import java.awt.*; +import java.awt.event.*; + +public class DiagGUI extends Frame { + public Counters counterWin; + public static void main(String args[]) { + new DiagGUI(); + } + + /** show counter window. creates it if not already present */ + public void showCounters() { + if(counterWin == null) { + counterWin = new Counters(); + } else { + counterWin.setVisible(true); + counterWin.toFront(); + } + } + + /** initialize the GUI. */ + public DiagGUI(){ + MenuItem item; + MenuBar menuBar = new MenuBar(); + Menu fileMenu = new Menu("File"); + item = new MenuItem("Exit"); + item.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + System.exit(0); + } + }); + fileMenu.add(item); + menuBar.add(fileMenu); + + Menu viewMenu = new Menu("View"); + item = new MenuItem("Counters"); + item.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + showCounters(); + } + }); + viewMenu.add(item); + menuBar.add(viewMenu); + + addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e){ + System.exit(0); + } + }); + setMenuBar(menuBar); + setSize(300,400); + setVisible(true); + } +} diff --git a/java/com/rsyslog/gui/msggen/MsgGen.java b/java/com/rsyslog/gui/msggen/MsgGen.java new file mode 100644 index 00000000..c57027ff --- /dev/null +++ b/java/com/rsyslog/gui/msggen/MsgGen.java @@ -0,0 +1,140 @@ +/* A yet very simple syslog message generator + * + * The purpose of this program is to provide a facility that enables + * to generate complex traffic patterns for testing purposes. It still is + * in its infancy, but hopefully will evolve. + * + * Note that this has been created as a stand-alone application because it + * was considered useful to have it as a separate program. But it should still + * be possible to call its class from any other program, specifically the debug + * GUI. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.msggen; +import com.rsyslog.lib.*; +import java.awt.*; +import java.awt.event.*; +import javax.swing.*; + +public class MsgGen extends Frame { + private TextField target; + private TextField message; + private TextField nummsgs; + private TextField numthrds; + + public static void main(String args[]) { + new MsgGen(); + } + + /** creates the menu bar INCLUDING all menu handlers */ + private void createMenu() { + MenuItem item; + MenuBar menuBar = new MenuBar(); + Menu fileMenu = new Menu("File"); + item = new MenuItem("Exit"); + item.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + System.exit(0); + } + }); + fileMenu.add(item); + menuBar.add(fileMenu); + + addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e){ + System.exit(0); + } + }); + setMenuBar(menuBar); + } + + /** creates the main GUI */ + private void createGUI() { + //target = new TextField("127.0.0.1", 32); + target = new TextField("172.19.3.3", 32); + message = new TextField(80); + //message.setText("<161>Test malformed"); + message.setText("<5>iaalog[171652]: AIB|dcu|2009/08/12 14:48:43|mfa challenge|NNNNNNN|XX.XX.XX.XX"); + nummsgs = new TextField("1000", 8); + numthrds = new TextField("10", 5); + Panel pCenter = new Panel(); + + Panel pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Target Host:")); + pnl.add(target); + pCenter.add(pnl); + + pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Number of Msgs:")); + pnl.add(nummsgs); + pCenter.add(pnl); + + pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Msg:")); + pnl.add(message); + pCenter.add(pnl); + + Panel pSouth = new Panel(); + pSouth.setLayout(new FlowLayout()); + + pnl = new Panel(); + pnl.setLayout(new FlowLayout()); + pnl.add(new Label("Number of Threads:")); + pnl.add(numthrds); + pSouth.add(pnl); + + Button b = new Button("Start Test"); + b.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + performTest(); + } + }); + pSouth.add(b); + + pack(); + setTitle("Syslog Message Generator"); + setLayout(new BorderLayout()); + add(pCenter, BorderLayout.CENTER); + add(pSouth, BorderLayout.SOUTH); + setSize(800,400); + } + + /** perform the test, a potentially complex operation */ + private void performTest() { + for(short i = 0 ; i < Integer.parseInt(numthrds.getText()) ; ++ i) { + SyslogTrafficGenerator gen = + new SyslogTrafficGenerator(target.getText(), message.getText(), + Long.parseLong(nummsgs.getText())); + gen.start(); + } + } + + + /** initialize the GUI. */ + public MsgGen(){ + createMenu(); + createGUI(); + setVisible(true); + } +} diff --git a/java/com/rsyslog/gui/simpServ/simpServ.java b/java/com/rsyslog/gui/simpServ/simpServ.java new file mode 100644 index 00000000..2a83dad0 --- /dev/null +++ b/java/com/rsyslog/gui/simpServ/simpServ.java @@ -0,0 +1,45 @@ +/** + * Implementation of a tcp-based syslog server. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ + +package com.rsyslog.gui.simpServ; +import com.rsyslog.lib.*; +//import com.rsyslog.gui.*; + +public class simpServ { + + public static void main(String args[]) { + try { + simpServConsumer cons = new simpServConsumer(); + System.out.println("Starting server on port " + args[0] + "\n"); + SyslogServerTCP myServ = new + SyslogServerTCP(Integer.parseInt(args[0]), cons); + myServ.start(); + System.out.println("Press ctl-c to terminate\n"); + } + catch(Exception e) { + System.out.println("Error: " + e.toString()); + } + } +} diff --git a/java/com/rsyslog/gui/simpServ/simpServConsumer.java b/java/com/rsyslog/gui/simpServ/simpServConsumer.java new file mode 100644 index 00000000..588f2640 --- /dev/null +++ b/java/com/rsyslog/gui/simpServ/simpServConsumer.java @@ -0,0 +1,32 @@ +/** A syslog message consumer for the simple syslog server. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.gui.simpServ; +import com.rsyslog.lib.*; + +class simpServConsumer implements SyslogMsgConsumer { + public void consumeMsg(String ln) { + SyslogMessage msg = new SyslogMessage(ln); + System.out.println("Line received '" + msg.getRawMsgAfterPRI() + "'\n"); + } +} diff --git a/java/com/rsyslog/lib/DiagSess.java b/java/com/rsyslog/lib/DiagSess.java new file mode 100644 index 00000000..799b9a4a --- /dev/null +++ b/java/com/rsyslog/lib/DiagSess.java @@ -0,0 +1,78 @@ +/* The diagnostic session to an imdiag module (running inside rsyslogd). + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; +import java.io.*; +import java.net.*; + +public class DiagSess { + + private String host = new String("127.0.0.1"); + private int port = 13500; + int timeout = 0; + private PrintWriter out = null; + private BufferedReader in = null; + private Socket diagSocket = null; + + /** set connection timeout */ + public void setTimeout(int timeout_) { + timeout = timeout_; + } + + public DiagSess(String host_, int port_) { + host = host_; + port = port_; + } + + /** connect to remote server. Initializes everything for request-reply + * processing. + * + * @throws IOException + */ + public void connect() throws IOException { + diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(timeout); + out = new PrintWriter(diagSocket.getOutputStream(), true); + in = new BufferedReader(new InputStreamReader( + diagSocket.getInputStream())); + + } + + /** end session with remote server. */ + public void disconnect() throws IOException { + out.close(); + in.close(); + diagSocket.close(); + } + + /** issue a request to imdiag and return its response. + * + * @param req request string + * @return response string (unparsed) + * @throws IOException + */ + public String request(String req) throws IOException { + out.println(req); + String resp = in.readLine(); + return resp; + } + +} diff --git a/java/com/rsyslog/lib/SyslogMessage.java b/java/com/rsyslog/lib/SyslogMessage.java new file mode 100644 index 00000000..b544a6db --- /dev/null +++ b/java/com/rsyslog/lib/SyslogMessage.java @@ -0,0 +1,75 @@ +/** + * Implementation of the syslog message object. + * + * This is a limited-capability implementation of a syslog message together + * with all its properties. It is limit to what is currently needed and may + * be extended as further need arises. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public class SyslogMessage { + + /** message as received from the wire */ + private String rawmsg; + /** the rawmsg without the PRI part */ + private String rawMsgAfterPRI; + /** PRI part */ + private int pri; + + /** a very simple syslog parser. So far, it only parses out the + * PRI part of the message. May be extended later. Rawmsg must have + * been set before the parser is called. It will populate "all" other + * fields. + */ + private void parse() { + int i; + if(rawmsg.charAt(0) == '<') { + pri = 0; + for(i = 1 ; Character.isDigit(rawmsg.charAt(i)) && i < 4 ; ++i) { + pri = pri * 10 + rawmsg.charAt(i) - '0'; + } + if(rawmsg.charAt(i) != '>') + /* not a real cure, but sufficient for the current + * mini-parser... */ + --i; + rawMsgAfterPRI = rawmsg.substring(i + 1); + } else { + pri = 116; + rawMsgAfterPRI = rawmsg; + } + } + + public SyslogMessage(String _rawmsg) { + rawmsg = _rawmsg; + parse(); + } + + public String getRawMsg() { + return rawmsg; + } + + public String getRawMsgAfterPRI() { + return rawMsgAfterPRI; + } +} diff --git a/java/com/rsyslog/lib/SyslogMsgConsumer.java b/java/com/rsyslog/lib/SyslogMsgConsumer.java new file mode 100644 index 00000000..42c9931a --- /dev/null +++ b/java/com/rsyslog/lib/SyslogMsgConsumer.java @@ -0,0 +1,29 @@ +/** + * Interface for SyslogConsumers. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public interface SyslogMsgConsumer { + public void consumeMsg(String msg); +} diff --git a/java/com/rsyslog/lib/SyslogSender.java b/java/com/rsyslog/lib/SyslogSender.java new file mode 100644 index 00000000..fc0e3fec --- /dev/null +++ b/java/com/rsyslog/lib/SyslogSender.java @@ -0,0 +1,96 @@ +/** + * This class specifies all methods common to syslog senders. It also implements + * some generic ways to send data. Actual syslog senders (e.g. UDP, TCP, ...) shall + * be derived from this class. + * + * This is a limited-capability implementation of a syslog message together + * with all its properties. It is limit to what is currently needed and may + * be extended as further need arises. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public abstract class SyslogSender { + + /** the rawmsg without the PRI part */ + private String target; + + /** the rawmsg without the PRI part */ + private boolean isConnected = false; + + /** Constructs Sender, sets target system. + * @param target the system to connect to. Syntax of target is depending + * on the underlying transport. + */ + public SyslogSender(String target) { + this.target = target; + } + + + /** send a message on the wire. + * This needs a complete formatted message, which will be extended by + * the transport framing, if necessary. + * + * @param MSG a validly formatted syslog message as of the RFC (all parts) + * @throws Exception (depending on transport) + */ + protected abstract void sendTransport(String MSG) throws Exception; + + /** send an alread-formatted message. + * Sends a preformatted syslog message payload to the target. Connects + * to the target if not already connected. + * + * @param MSG a validly formatted syslog message as of the RFC (all parts) + * @throws Exception (depending on transport) + */ + public void sendMSG(String MSG) throws Exception { + if(!isConnected) + connect(); + sendTransport(MSG); + } + + /** connect to the target. + * Note that this may be a null operation if there is no session-like entity + * in the underlying transport (as is for example in UDP). + */ + public void connect() throws Exception { + /* the default implementation does (almost) nothing */ + isConnected = true; + } + + /** disconnects from the target. + * Note that this may be a null operation if there is no session-like entity + * in the underlying transport (as is for example in UDP). + */ + public void disconnect() { + /* the default implementation does (almost) nothing */ + isConnected = false; + } + + /** return target of this Sender. + * @returns target as initially set + */ + public String getTarget() { + return target; + } +} diff --git a/java/com/rsyslog/lib/SyslogServerTCP.java b/java/com/rsyslog/lib/SyslogServerTCP.java new file mode 100644 index 00000000..d5376a32 --- /dev/null +++ b/java/com/rsyslog/lib/SyslogServerTCP.java @@ -0,0 +1,126 @@ +/** + * Implementation of a tcp-based syslog server. + * + * This is a limited-capability implementation of a syslog tcp server. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +import com.rsyslog.lib.SyslogMsgConsumer; +import java.io.*; +import java.net.*; + + +/** a small test consumer */ +/* +class TestConsumer implements SyslogMsgConsumer { + public void consumeMsg(String ln) { + System.out.println("Line received '" + ln + "'\n"); + } +} +*/ + +public class SyslogServerTCP extends Thread { + private ServerSocket lstnSock; + private boolean contRun; /* continue processing requests? */ + public SyslogMsgConsumer consumer; + + /** Process a single connection */ + class Session extends Thread { + private Socket sock; + private SyslogServerTCP srvr; + + public Session(Socket so, SyslogServerTCP _srvr) { + sock = so; + srvr = _srvr; + } + + public void run() { + try { + BufferedReader data = new BufferedReader( + new InputStreamReader(sock.getInputStream())); + + String ln = data.readLine(); + while(ln != null) { + srvr.getConsumer().consumeMsg(ln); + ln = data.readLine(); + } + System.out.println("End of Session.\n"); + sock.close(); + } + catch(Exception e) { + /* we ignore any errors we may have... */ + System.out.println("Session exception " + e.toString()); + } + } + } + + /** a small test driver */ +/* + public static void main(String args[]) { + try { + SyslogMsgConsumer cons = new TestConsumer(); + System.out.println("Starting server on port " + args[0] + "\n"); + SyslogServerTCP myServ = new + SyslogServerTCP(Integer.parseInt(args[0]), cons); + myServ.start(); + System.out.println("Press ctl-c to terminate\n"); + } + catch(Exception e) { + System.out.println("Fehler! " + e.toString()); + } + } +*/ + + public SyslogServerTCP(int port, SyslogMsgConsumer cons) throws java.io.IOException { + if(lstnSock != null) + terminate(); + lstnSock = new ServerSocket(port); + consumer = cons; + contRun = true; + } + + public void terminate() { + contRun = false; + } + + public SyslogMsgConsumer getConsumer() { + return consumer; + } + + public void run() { + try { + while(contRun) { + Socket sock = lstnSock.accept(); + System.out.println("New connection request! " + sock.toString()); + Thread sess = new Session(sock, this); + sock = null; + sess.start(); + } + } + catch(Exception e) { + System.out.println("Error during server run " + e.toString()); + } + + } +} diff --git a/java/com/rsyslog/lib/SyslogTrafficGenerator.java b/java/com/rsyslog/lib/SyslogTrafficGenerator.java new file mode 100644 index 00000000..79a99495 --- /dev/null +++ b/java/com/rsyslog/lib/SyslogTrafficGenerator.java @@ -0,0 +1,81 @@ +/** + * This class is a syslog traffic generator. It is primarily intended to be used + * together with testing tools, but may have some use cases outside that domain. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; + +public class SyslogTrafficGenerator extends Thread { + + /** the target host to receive traffic */ + private String target; + + /** the message (template) to be sent */ + private String message; + + /** number of messages to be sent */ + private long nummsgs; + + /** Constructs Sender, sets target system. + * @param target the system to connect to. Syntax of target is depending + * on the underlying transport. + */ + public SyslogTrafficGenerator(String target, String message, long nummsgs) { + this.target = target; + this.message = message; + this.nummsgs = nummsgs; + } + + /** Generates the traffic. Stops when either called to terminate + * or the max number of messages have been sent. Note that all + * necessary properties must have been set up before starting the + * generator thread! + */ + private void performTest() throws Exception { + int doDisp = 0; + UDPSyslogSender sender = new UDPSyslogSender(target); + for(long i = 0 ; i < nummsgs ; ++i) { + sender.sendMSG(message + " " + Long.toString(i) + " " + this.toString() + "\0"); + if((doDisp++ % 1000) == 0) + System.out.println(this.toString() + " send message " + Long.toString(i)); + sleep(1); + } + } + + +/** Wrapper around the real traffic generator, catches exceptions. + */ + public void run() { + System.out.println("traffic generator " + this.toString() + " thread started"); + try { + performTest(); + } + catch(Exception e) { + /* at some time, we may find a more intelligent way to + * handle this! ;) + */ + System.out.println(e.toString()); + } + System.out.println("traffic generator " + this.toString() + " thread finished"); + } +} diff --git a/java/com/rsyslog/lib/UDPSyslogSender.java b/java/com/rsyslog/lib/UDPSyslogSender.java new file mode 100644 index 00000000..1a2c4726 --- /dev/null +++ b/java/com/rsyslog/lib/UDPSyslogSender.java @@ -0,0 +1,75 @@ +/** + * A UDP transport implementation of a syslog sender. + * + * Note that there is an anomaly in this version of the code: we query the remote system + * address only once during the connection setup and resue it. If we potentially run for + * an extended period of time, the remote address may change, what we do not reflect. For + * the current use case, this is acceptable, but if this code is put into more wide-spread + * use outside of debugging, a periodic requery should be added. + * + * @author Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see http://www.gnu.org/licenses/. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +package com.rsyslog.lib; +import java.net.*; + +public class UDPSyslogSender extends SyslogSender { + + private final int port = 514; // TODO: take from target! + private InetAddress targetAddr; + + /** the socket to communicate over with the remote system. */ + private DatagramSocket sock; + + /** Constructs Sender, sets target system. + * @param target the system to connect to. Syntax of target is depending + * on the underlying transport. + */ + public UDPSyslogSender(String target) throws Exception { + super(target); + } + + /** send a message on the wire. + * This needs a complete formatted message, which will be extended by + * the transport framing, if necessary. + * + * @param MSG a validly formatted syslog message as of the RFC (all parts) + * @throws Exception (depending on transport) + */ + protected void sendTransport(String MSG) throws Exception { + byte msg[] = MSG.getBytes(); + DatagramPacket pkt = new DatagramPacket(msg, msg.length, targetAddr, port); + sock.send(pkt); + } + + + /** connect to the target. + * For UDP, this means we create the socket. + */ + public void connect() throws Exception { + super.connect(); + sock = new DatagramSocket(); + + // TODO: we should extract the actual hostname & port! + targetAddr = InetAddress.getByName(getTarget()); + } + +} diff --git a/m4/atomic_operations_64bit.m4 b/m4/atomic_operations_64bit.m4 new file mode 100644 index 00000000..3121cf15 --- /dev/null +++ b/m4/atomic_operations_64bit.m4 @@ -0,0 +1,53 @@ +# rsyslog +# +# atomic_operations.m4 - autoconf macro to check if compiler supports atomic +# operations +# +# rgerhards, 2008-09-18, added based on +# http://svn.apache.org/repos/asf/apr/apr/trunk/configure.in +# +# +AC_DEFUN([RS_ATOMIC_OPERATIONS_64BIT], +[AC_CACHE_CHECK([whether the compiler provides atomic builtins for 64 bit data types], [ap_cv_atomic_builtins_64], +[AC_TRY_RUN([ +int main() +{ + unsigned long long val = 1010, tmp, *mem = &val; + + if (__sync_fetch_and_add(&val, 1010) != 1010 || val != 2020) + return 1; + + tmp = val; + + if (__sync_fetch_and_sub(mem, 1010) != tmp || val != 1010) + return 1; + + if (__sync_sub_and_fetch(&val, 1010) != 0 || val != 0) + return 1; + + tmp = 3030; + + if (__sync_val_compare_and_swap(mem, 0, tmp) != 0 || val != tmp) + return 1; + + if (__sync_lock_test_and_set(&val, 4040) != 3030) + return 1; + + mem = &tmp; + + if (__sync_val_compare_and_swap(&mem, &tmp, &val) != &tmp) + return 1; + + __sync_synchronize(); + + if (mem != &val) + return 1; + + return 0; +}], [ap_cv_atomic_builtins_64=yes], [ap_cv_atomic_builtins_64=no], [ap_cv_atomic_builtins_64=no])]) + +if test "$ap_cv_atomic_builtins_64" = "yes"; then + AC_DEFINE(HAVE_ATOMIC_BUILTINS_64BIT, 1, [Define if compiler provides 64 bit atomic builtins]) +fi + +]) diff --git a/m4/shave.m4 b/m4/shave.m4 deleted file mode 100644 index e647e579..00000000 --- a/m4/shave.m4 +++ /dev/null @@ -1,99 +0,0 @@ -dnl Make automake/libtool output more friendly to humans -dnl -dnl Copyright (c) 2009, Damien Lespiau <damien.lespiau@gmail.com> -dnl -dnl Permission is hereby granted, free of charge, to any person -dnl obtaining a copy of this software and associated documentation -dnl files (the "Software"), to deal in the Software without -dnl restriction, including without limitation the rights to use, -dnl copy, modify, merge, publish, distribute, sublicense, and/or sell -dnl copies of the Software, and to permit persons to whom the -dnl Software is furnished to do so, subject to the following -dnl conditions: -dnl -dnl The above copyright notice and this permission notice shall be -dnl included in all copies or substantial portions of the Software. -dnl -dnl THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -dnl EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -dnl OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -dnl NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -dnl HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -dnl WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -dnl FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -dnl OTHER DEALINGS IN THE SOFTWARE. -dnl -dnl SHAVE_INIT([shavedir],[default_mode]) -dnl -dnl shavedir: the directory where the shave scripts are, it defaults to -dnl $(top_builddir) -dnl default_mode: (enable|disable) default shave mode. This parameter -dnl controls shave's behaviour when no option has been -dnl given to configure. It defaults to disable. -dnl -dnl * SHAVE_INIT should be called late in your configure.(ac|in) file (just -dnl before AC_CONFIG_FILE/AC_OUTPUT is perfect. This macro rewrites CC and -dnl LIBTOOL, you don't want the configure tests to have these variables -dnl re-defined. -dnl * This macro requires GNU make's -s option. - -AC_DEFUN([_SHAVE_ARG_ENABLE], -[ - AC_ARG_ENABLE([shave], - AS_HELP_STRING( - [--enable-shave], - [use shave to make the build pretty [[default=$1]]]),, - [enable_shave=$1] - ) -]) - -AC_DEFUN([SHAVE_INIT], -[ - dnl you can tweak the default value of enable_shave - m4_if([$2], [enable], [_SHAVE_ARG_ENABLE(yes)], [_SHAVE_ARG_ENABLE(no)]) - - if test x"$enable_shave" = xyes; then - dnl where can we find the shave scripts? - m4_if([$1],, - [shavedir="$ac_pwd"], - [shavedir="$ac_pwd/$1"]) - AC_SUBST(shavedir) - - dnl make is now quiet - AC_SUBST([MAKEFLAGS], [-s]) - AC_SUBST([AM_MAKEFLAGS], ['`test -z $V && echo -s`']) - - dnl we need sed - AC_CHECK_PROG(SED,sed,sed,false) - - dnl substitute libtool - SHAVE_SAVED_LIBTOOL=$LIBTOOL - LIBTOOL="${SHELL} ${shavedir}/shave-libtool '${SHAVE_SAVED_LIBTOOL}'" - AC_SUBST(LIBTOOL) - - dnl substitute cc/cxx - SHAVE_SAVED_CC=$CC - SHAVE_SAVED_CXX=$CXX - SHAVE_SAVED_FC=$FC - SHAVE_SAVED_F77=$F77 - SHAVE_SAVED_OBJC=$OBJC - CC="${SHELL} ${shavedir}/shave cc ${SHAVE_SAVED_CC}" - CXX="${SHELL} ${shavedir}/shave cxx ${SHAVE_SAVED_CXX}" - FC="${SHELL} ${shavedir}/shave fc ${SHAVE_SAVED_FC}" - F77="${SHELL} ${shavedir}/shave f77 ${SHAVE_SAVED_F77}" - OBJC="${SHELL} ${shavedir}/shave objc ${SHAVE_SAVED_OBJC}" - AC_SUBST(CC) - AC_SUBST(CXX) - AC_SUBST(FC) - AC_SUBST(F77) - AC_SUBST(OBJC) - - V=@ - else - V=1 - fi - Q='$(V:1=)' - AC_SUBST(V) - AC_SUBST(Q) -]) - diff --git a/outchannel.c b/outchannel.c index 74c18218..b9b4a6b1 100644 --- a/outchannel.c +++ b/outchannel.c @@ -38,6 +38,7 @@ #include "stringbuf.h" #include "outchannel.h" #include "dirty.h" +#include "debug.h" static struct outchannel *ochRoot = NULL; /* the root of the outchannel list */ static struct outchannel *ochLast = NULL; /* points to the last element of the outchannel list */ @@ -212,7 +213,7 @@ struct outchannel *ochAddLine(char* pName, uchar** ppRestOfConfLine) return NULL; pOch->iLenName = strlen(pName); - pOch->pszName = (char*) malloc(sizeof(char) * (pOch->iLenName + 1)); + pOch->pszName = (char*) MALLOC(sizeof(char) * (pOch->iLenName + 1)); if(pOch->pszName == NULL) { dbgprintf("ochAddLine could not alloc memory for outchannel name!"); pOch->iLenName = 0; @@ -37,6 +37,7 @@ #include "rsyslog.h" #include "net.h" /* struct NetAddr */ #include "parse.h" +#include "debug.h" /* ################################################################# * * private members * @@ -429,7 +430,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) switch(getaddrinfo ((char*)pszIP+1, NULL, &hints, &res)) { case 0: - (*pIP)->addr.NetAddr = malloc (res->ai_addrlen); + (*pIP)->addr.NetAddr = MALLOC (res->ai_addrlen); memcpy ((*pIP)->addr.NetAddr, res->ai_addr, res->ai_addrlen); freeaddrinfo (res); break; @@ -468,7 +469,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) switch(getaddrinfo ((char*)pszIP, NULL, &hints, &res)) { case 0: - (*pIP)->addr.NetAddr = malloc (res->ai_addrlen); + (*pIP)->addr.NetAddr = MALLOC (res->ai_addrlen); memcpy ((*pIP)->addr.NetAddr, res->ai_addr, res->ai_addrlen); freeaddrinfo (res); break; diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index bf972191..81b357ef 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -213,7 +213,6 @@ doInjectMsg(int iNum) MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pRcvDummy); CHKiRet(MsgSetRcvFromIP(pMsg, pRcvIPDummy)); CHKiRet(submitMsg(pMsg)); @@ -246,7 +245,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess) doInjectMsg(i + iFrom); } - CHKiRet(sendResponse(pSess, "messages injected\n")); + CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); finalize_it: RETiRet; @@ -259,12 +258,23 @@ static rsRetVal waitMainQEmpty(tcps_sess_t *pSess) { int iMsgQueueSize; + int iPrint = 0; DEFiRet; CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); while(iMsgQueueSize > 0) { + /* DEV DEBUG ONLY if(iPrint++ % 500) + printf("imdiag: main msg queue size: %d\n", iMsgQueueSize); + */ + if(iPrint++ % 500 == 0) + dbgprintf("imdiag sleeping, wait mainq drain, curr size %d\n", iMsgQueueSize); srSleep(0,2); /* wait a little bit */ CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + if(iMsgQueueSize == 0) { + /* verify that queue is still empty (else it could just be a race!) */ + srSleep(1,5); /* wait a little bit */ + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + } } CHKiRet(sendResponse(pSess, "mainqueue empty\n")); @@ -291,12 +301,13 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) * WITHOUT a termination \0 char. So we need to convert it to one * before proceeding. */ - CHKmalloc(pszMsg = malloc(sizeof(uchar) * (iLenMsg + 1))); + CHKmalloc(pszMsg = MALLOC(sizeof(uchar) * (iLenMsg + 1))); memcpy(pszMsg, pRcv, iLenMsg); pszMsg[iLenMsg] = '\0'; getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf)/sizeof(uchar), TO_LOWERCASE); + dbgprintf("imdiag received command '%s'\n", cmdBuf); if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); @@ -443,10 +454,17 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus } +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 7c588f90..8a10e26f 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -5,7 +5,7 @@ * * Work originally begun on 2008-02-01 by Rainer Gerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -107,7 +107,6 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); - pMsg->bParseHOSTNAME = 0; CHKiRet(submitMsg(pMsg)); finalize_it: RETiRet; @@ -214,7 +213,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) } finalize_it: - /*EMPTY - just to keep the compiler happy, do NOT remove*/; + ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ /* Note: the problem above is that pthread:cleanup_pop() is a macro which * evaluates to something like "} while(0);". So the code would become * "finalize_it: }", that is a label without a statement. The C standard does @@ -244,27 +243,12 @@ finalize_it: * IMPORTANT: the calling interface of this function can NOT be modified. It actually is * called by pthreads. The provided argument is currently not being used. */ -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ static void inputModuleCleanup(void __attribute__((unused)) *arg) { BEGINfunc -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ - - - - /* so far not needed */ - - - -/* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ ENDfunc } -/* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ /* This function is called by the framework to gather the input. The module stays @@ -292,28 +276,22 @@ BEGINrunInput int i; int bHadFileData; /* were there at least one file with data during this run? */ CODESTARTrunInput - /* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ pthread_cleanup_push(inputModuleCleanup, NULL); - while(1) { /* endless loop - do NOT break; out of it! */ - /* END no-touch zone * - * ------------------------------------------------------------------------------------------ */ + while(1) { - do { - bHadFileData = 0; - for(i = 0 ; i < iFilPtr ; ++i) { - pollFile(&files[i], &bHadFileData); - } - } while(iFilPtr > 1 && bHadFileData == 1); /* waring: do...while()! */ + do { + bHadFileData = 0; + for(i = 0 ; i < iFilPtr ; ++i) { + pollFile(&files[i], &bHadFileData); + } + } while(iFilPtr > 1 && bHadFileData == 1); /* warning: do...while()! */ - /* Note: the additional 10ns wait is vitally important. It guards rsyslog against totally - * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any - * other valid scenario. So do not remove. -- rgerhards, 2008-02-14 - */ - srSleep(iPollInterval, 10); + /* Note: the additional 10ns wait is vitally important. It guards rsyslog against totally + * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any + * other valid scenario. So do not remove. -- rgerhards, 2008-02-14 + */ + srSleep(iPollInterval, 10); - /* ------------------------------------------------------------------------------------------ * - * DO NOT TOUCH the following code - it will soon be part of the module generation macros! */ } /*NOTREACHED*/ diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index 1aad6622..dd3d67e3 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -48,6 +48,7 @@ #include "dirty.h" #include "cfsysline.h" #include "module-template.h" +#include "unicode-helper.h" #include "net.h" #include "srUtils.h" #include "gss-misc.h" @@ -56,6 +57,7 @@ #include "errmsg.h" #include "netstrm.h" #include "glbl.h" +#include "debug.h" #include "unlimited_select.h" @@ -177,10 +179,10 @@ isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void *pUsrSrv, void*p pGSess = (gss_sess_t*) pUsrSess; if((pGSrv->allowedMethods & ALLOWEDMETHOD_TCP) && - net.isAllowedSender((uchar*)"TCP", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"TCP", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_TCP; if((pGSrv->allowedMethods & ALLOWEDMETHOD_GSS) && - net.isAllowedSender((uchar*)"GSS", addr, (char*)fromHostFQDN)) + net.isAllowedSender2((uchar*)"GSS", addr, (char*)fromHostFQDN, 1)) allowedMethods |= ALLOWEDMETHOD_GSS; if(allowedMethods && pGSess != NULL) pGSess->allowedMethods = allowedMethods; @@ -331,6 +333,7 @@ addGSSListener(void __attribute__((unused)) *pVal, uchar *pNewVal) CHKiRet(tcpsrv.SetCBOnSessAccept(pOurTcpsrv, onSessAccept)); CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, UCHAR_CONSTANT("imgssapi"))); tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); } @@ -408,7 +411,7 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) */ char *buf; int ret = 0; - CHKmalloc(buf = (char*) malloc(sizeof(char) * (glbl.GetMaxLine() + 1))); + CHKmalloc(buf = (char*) MALLOC(sizeof(char) * (glbl.GetMaxLine() + 1))); dbgprintf("GSS-API Trying to accept TCP session %p\n", pSess); @@ -682,9 +685,17 @@ CODESTARTafterRun ENDafterRun +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index 6d7b6c98..b7899353 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -75,6 +75,7 @@ #include "rsyslog.h" #include "imklog.h" +#include "debug.h" /* globals */ static int fklog = -1; /* /dev/klog */ @@ -132,7 +133,7 @@ readklog(void) if((size_t) iMaxLine < sizeof(bufRcv) - 1) { pRcv = bufRcv; } else { - if((pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))) == NULL) + if((pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))) == NULL) iMaxLine = sizeof(bufRcv) - 1; /* better this than noting */ } diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 7994c5eb..c59ce04f 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -111,7 +111,6 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); pMsg->iFacility = LOG_FAC(iFacility); pMsg->iSeverity = LOG_PRI(iSeverity); - pMsg->bParseHOSTNAME = 0; CHKiRet(submitMsg(pMsg)); finalize_it: diff --git a/plugins/imklog/ksym.c b/plugins/imklog/ksym.c index f636a7bb..058b2cfa 100644 --- a/plugins/imklog/ksym.c +++ b/plugins/imklog/ksym.c @@ -122,6 +122,7 @@ #include "imklog.h" #include "ksyms.h" #include "module.h" +#include "debug.h" int num_syms = 0; @@ -523,7 +524,7 @@ static int AddSymbol(unsigned long address, char *symbol) return(0); /* Then the space for the symbol. */ - sym_array[num_syms].name = (char *) malloc(strlen(symbol)*sizeof(char) + 1); + sym_array[num_syms].name = (char *) MALLOC(strlen(symbol)*sizeof(char) + 1); if ( sym_array[num_syms].name == NULL ) return(0); diff --git a/plugins/imklog/ksym_mod.c b/plugins/imklog/ksym_mod.c index be5fdee9..82978892 100644 --- a/plugins/imklog/ksym_mod.c +++ b/plugins/imklog/ksym_mod.c @@ -106,6 +106,7 @@ #include "rsyslog.h" #include "imklog.h" #include "ksyms.h" +#include "debug.h" #define KSYMS "/proc/kallsyms" @@ -289,7 +290,7 @@ struct Module *AddModule(module) struct Module *mp; if ( num_modules == 0 ) { - sym_array_modules = (struct Module *)malloc(sizeof(struct Module)); + sym_array_modules = (struct Module *)MALLOC(sizeof(struct Module)); if ( sym_array_modules == NULL ) { diff --git a/plugins/immark/immark.c b/plugins/immark/immark.c index 8504f872..5d48369e 100644 --- a/plugins/immark/immark.c +++ b/plugins/immark/immark.c @@ -42,6 +42,8 @@ #include "module-template.h" #include "errmsg.h" #include "msg.h" +#include "srUtils.h" +#include "glbl.h" MODULE_TYPE_INPUT @@ -50,8 +52,16 @@ MODULE_TYPE_INPUT /* Module static data */ DEF_IMOD_STATIC_DATA +DEFobjCurrIf(glbl) static int iMarkMessagePeriod = DEFAULT_MARK_PERIOD; +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + /* This function is called to gather input. It must terminate only * a) on failure (iRet set accordingly) * b) on termination of the input module (as part of the unload process) @@ -71,16 +81,13 @@ CODESTARTrunInput * right into the sleep below. */ while(1) { - /* we do not need to handle the RS_RET_TERMINATE_NOW case any - * special because we just need to terminate. This may be different - * if a cleanup is needed. But for now, we can just use CHKiRet(). - * rgerhards, 2007-12-17 - */ - CHKiRet(thrdSleep(pThrd, iMarkMessagePeriod, 0)); /* seconds, micro seconds */ + srSleep(iMarkMessagePeriod, 0); /* seconds, micro seconds */ + + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + logmsgInternal(NO_ERRCODE, LOG_INFO, (uchar*)"-- MARK --", MARK); } -finalize_it: - return iRet; ENDrunInput @@ -106,6 +113,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) @@ -119,9 +127,9 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"markmessageperiod", 0, eCmdHdlrInt, NULL, &iMarkMessagePeriod, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/plugins/imsolaris/imsolaris.c b/plugins/imsolaris/imsolaris.c index 6b07ba2b..f801833b 100644 --- a/plugins/imsolaris/imsolaris.c +++ b/plugins/imsolaris/imsolaris.c @@ -205,7 +205,6 @@ readLog(int fd, uchar *pRcv, int iMaxLine) MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); pMsg->iFacility = LOG_FAC(hdr.pri); pMsg->iSeverity = LOG_PRI(hdr.pri); - pMsg->bParseHOSTNAME = 0; pMsg->msgFlags = NEEDS_PARSING | NO_PRI_IN_RAW; CHKiRet(submitMsg(pMsg)); } @@ -224,7 +223,7 @@ finalize_it: * rgerhards, 2010-04-19 */ static inline rsRetVal -getMsgs(int timeout) +getMsgs(thrdInfo_t *pThrd, int timeout) { DEFiRet; int nfds; @@ -247,12 +246,14 @@ getMsgs(int timeout) CHKmalloc(pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))); } - do { + while(pThrd->bShallStop != TRUE) { DBGPRINTF("imsolaris: waiting for next message (timeout %d)...\n", timeout); if(timeout == 0) { nfds = poll(&sun_Pfd, 1, timeout); /* wait without timeout */ - /* v5-TODO: here we must check if we should terminante! */ + if(pThrd->bShallStop == TRUE) { + break; + } if(nfds == 0) { if(timeout == 0) { @@ -290,9 +291,7 @@ getMsgs(int timeout) readLog(sun_Pfd.fd, pRcv, iMaxLine); } - } while(1); /* TODO: in v5, we must check the termination predicate */ - - /* Note: in v4, this code is never reached (our thread will be cancelled) */ + } finalize_it: if(pRcv != NULL && (size_t) iMaxLine >= sizeof(bufRcv) - 1) @@ -311,7 +310,7 @@ CODESTARTrunInput */ DBGPRINTF("imsolaris: doing startup poll before openeing door()\n"); - CHKiRet(getMsgs(0)); + CHKiRet(getMsgs(pThrd, 0)); /* note: sun's syslogd code claims that the door should only * be opened when the log stream has been polled. So file header @@ -319,8 +318,9 @@ CODESTARTrunInput */ sun_open_door(); DBGPRINTF("imsolaris: starting regular poll loop\n"); - iRet = getMsgs(-1); /* this is the primary poll loop, infinite timeout */ + iRet = getMsgs(pThrd, -1); /* this is the primary poll loop, infinite timeout */ + DBGPRINTF("imsolaris: terminating (bShallStop=%d)\n", pThrd->bShallStop); finalize_it: RETiRet; ENDrunInput @@ -359,9 +359,17 @@ CODESTARTmodExit ENDmodExit +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index d122e976..0cfae057 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -86,6 +86,7 @@ static int iTCPLstnMax = 20; /* max number of sessions */ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ static int bEmitMsgOnClose = 0; /* emit an informational message on close by remote peer */ static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ +static int bDisableLFDelim = 0; /* disbale standard LF delimiter */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ @@ -97,7 +98,7 @@ static int isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) { - return net.isAllowedSender(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN); + return net.isAllowedSender2(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN, 1); } @@ -171,7 +172,7 @@ static rsRetVal setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) localRet = ruleset.GetRuleset(&pRuleset, pszName); if(localRet == RS_RET_NOT_FOUND) { - errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + errmsg.LogError(0, RS_RET_RULESET_NOT_FOUND, "error: ruleset '%s' not found - ignored", pszName); } CHKiRet(localRet); pBindRuleset = pRuleset; @@ -198,6 +199,7 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); + CHKiRet(tcpsrv.SetbDisableLFDelim(pOurTcpsrv, bDisableLFDelim)); CHKiRet(tcpsrv.SetNotificationOnRemoteClose(pOurTcpsrv, bEmitMsgOnClose)); /* now set optional params, but only if they were actually configured */ if(pszStrmDrvrAuthMode != NULL) { @@ -254,6 +256,13 @@ CODESTARTafterRun ENDafterRun +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINmodExit CODESTARTmodExit if(pOurTcpsrv != NULL) @@ -281,6 +290,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus iStrmDrvrMode = 0; bEmitMsgOnClose = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + bDisableLFDelim = 0; free(pszInputName); pszInputName = NULL; free(pszStrmDrvrAuthMode); @@ -293,6 +303,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt @@ -326,6 +337,8 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserveraddtlframedelimiter"), 0, eCmdHdlrInt, NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverdisablelfdelimiter"), 0, eCmdHdlrBinary, + NULL, &bDisableLFDelim, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverinputname"), 0, eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverbindruleset"), 0, diff --git a/plugins/imtemplate/imtemplate.c b/plugins/imtemplate/imtemplate.c index 366408a0..e5e43025 100644 --- a/plugins/imtemplate/imtemplate.c +++ b/plugins/imtemplate/imtemplate.c @@ -77,6 +77,7 @@ #include "cfsysline.h" /* access to config file objects */ #include "module-template.h" /* generic module interface code - very important, read it! */ #include "srUtils.h" /* some utility functions */ +#include "debug.h" /* some debug helper functions */ MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -137,7 +138,7 @@ imtemplateMyFunc(int iMyParam) * ABORT_FINALIZE(retcode) * just like FINALIZE, except that iRet is set to the provided error * code before control is transferred, e.g. - * if((ptr = malloc(20)) == NULL) + * if((ptr = MALLOC(20)) == NULL) * ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); * * In order for all this to work, you need to define finalize_it, e.g. @@ -231,49 +232,25 @@ CODESTARTrunInput * logs an error message as syslogd, just as printf, e.g. * errmsg.LogError(NO_ERRCODE, "Error %d occured during %s", 1, "test"); * - * There are several ways how a message can be enqueued. This part of the - * interface is currently underspecified. Have a look at the function definitions - * in syslogd.c (sorry, folks...). - * - * If you received a full syslog message that must be decoded by a message - * parser, parseAndSubmitMessage() is the way to go. It's not just a funny name - * but also a quite some legacy. Consequently, its interface is, ummm, not - * well designed. - * parseAndSubmitMessage((char*)fromHost, (char*) pRcvBuf, lenRcvd, bParseHost); - * fromHost - * is the host that we received the message from (a string) - * pRcvBuf - * is the received (to-be-decoded) message - * lenRcvd - * is the length of the received message. Please note that pRcvBuf is - * NOT a standard C-string. Most importantly it is NOT expected to be - * \0-terminated. Thus the lenght is vitally imporant (if it is wrong, - * rsyslogd will probably segfault). - * bParseHost - * is a boolean (0-no, 1-yes). It tells the parser whether or not - * a hostname should be parsed from the message. This is important - * for sources that are known not to provide a hostname. - * Use define MSG_PARSE_HOSTNAME and MSG_DONT_PARSE_HOSTNAME - * - * Another, more elaborate, way is to create the message object ourselves and - * pass it to the rule engine. That way is more appropriate if the message + * To submit the message to the queue engine, we must create the message + * object and fill it with data. If it contains a syslog message that must + * be parsed, we can add a flag that requests parsing. Otherwise, we must + * fill the properties ourselves. That is appropriate if the message * does not need to be parsed, for example when reading text (log) files. In that way, * we can set the message properties as of our liking. This is how it works: * msg_t *pMsg; CHKiRet(msgConstruct(&pMsg)); - MsgSetUxTradMsg(pMsg, msg); MsgSetRawMsg(pMsg, msg); MsgSetHOSTNAME(pMsg, LocalHostName); MsgSetTAG(pMsg, "rsyslogd:"); pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - pMsg->bParseHOSTNAME = 0; flags |= INTERNAL_MSG; logmsg(pMsg, flags); / * some time, CHKiRet() will work here, too [today NOT!] * / * - * Note that UxTradMsg is a wild construct. For the time being, set it to - * the raw message text. I am hard thinking at dropping that beast at all... + * NOTE: for up-to-date usage samples, see the other provided input modules. + * A good starting point is probably imuxsock. * * This example probably does not set all message properties (but the ones * that are of practical importance). If you need all, check msg.h. Use @@ -314,7 +291,7 @@ CODESTARTwillRun if(udpLstnSocks == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); - if((pRcvBuf = malloc(glbl.GetMaxLine * sizeof(char))) == NULL) { + if((pRcvBuf = MALLOC(glbl.GetMaxLine * sizeof(char))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } * diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index d76f3544..99b69731 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -32,6 +32,9 @@ #include <errno.h> #include <unistd.h> #include <netdb.h> +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif #include "rsyslog.h" #include "dirty.h" #include "net.h" @@ -44,8 +47,8 @@ #include "parser.h" #include "datetime.h" #include "prop.h" +#include "ruleset.h" #include "unicode-helper.h" -#include "unlimited_select.h" MODULE_TYPE_INPUT @@ -58,7 +61,9 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(net) DEFobjCurrIf(datetime) DEFobjCurrIf(prop) +DEFobjCurrIf(ruleset) +static int bDoACLCheck; /* are ACL checks neeed? Cached once immediately before listener startup */ static int iMaxLine; /* maximum UDP message size supported */ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitted sender was last discarded * This shall prevent remote DoS when the "discard on disallowed sender" @@ -66,13 +71,14 @@ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitte */ static int *udpLstnSocks = NULL; /* Internet datagram sockets, first element is nbr of elements * read-only after init(), but beware of restart! */ +static ruleset_t **udpRulesets = NULL; /* ruleset to be used with sockets in question (entry 0 is empty) */ static uchar *pszBindAddr = NULL; /* IP to bind socket to */ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a global and alloc * it so that we can check available memory in willRun() and request * termination if we can not get it. -- rgerhards, 2007-12-27 */ static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ -// TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ +static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -91,6 +97,7 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) int *newSocks; int *tmpSocks; int iSrc, iDst; + ruleset_t **tmpRulesets; /* check which address to bind to. We could do this more compact, but have not * done so in order to make the code more readable. -- rgerhards, 2007-12-27 @@ -111,26 +118,39 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) if(udpLstnSocks == NULL) { /* esay, we can just replace it */ udpLstnSocks = newSocks; + CHKmalloc(udpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (newSocks[0] + 1))); + for(iDst = 1 ; iDst <= newSocks[0] ; ++iDst) + udpRulesets[iDst] = pBindRuleset; } else { /* we need to add them */ - if((tmpSocks = malloc(sizeof(int) * (1 + newSocks[0] + udpLstnSocks[0]))) == NULL) { - dbgprintf("out of memory trying to allocate udp listen socket array\n"); + tmpSocks = (int*) MALLOC(sizeof(int) * (1 + newSocks[0] + udpLstnSocks[0])); + tmpRulesets = (ruleset_t**) MALLOC(sizeof(ruleset_t*) * (1 + newSocks[0] + udpLstnSocks[0])); + if(tmpSocks == NULL || tmpRulesets == NULL) { + DBGPRINTF("out of memory trying to allocate udp listen socket array\n"); /* in this case, we discard the new sockets but continue with what we * already have */ free(newSocks); + free(tmpSocks); + free(tmpRulesets); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } else { /* ready to copy */ iDst = 1; - for(iSrc = 1 ; iSrc <= udpLstnSocks[0] ; ++iSrc) - tmpSocks[iDst++] = udpLstnSocks[iSrc]; - for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc) - tmpSocks[iDst++] = newSocks[iSrc]; + for(iSrc = 1 ; iSrc <= udpLstnSocks[0] ; ++iSrc, ++iDst) { + tmpSocks[iDst] = udpLstnSocks[iSrc]; + tmpRulesets[iDst] = udpRulesets[iSrc]; + } + for(iSrc = 1 ; iSrc <= newSocks[0] ; ++iSrc, ++iDst) { + tmpSocks[iDst] = newSocks[iSrc]; + tmpRulesets[iDst] = pBindRuleset; + } tmpSocks[0] = udpLstnSocks[0] + newSocks[0]; free(newSocks); free(udpLstnSocks); udpLstnSocks = tmpSocks; + free(udpRulesets); + udpRulesets = tmpRulesets; } } } @@ -142,7 +162,6 @@ finalize_it: } -#if 0 /* TODO: implement when tehre is time, requires restructure of socket array! */ /* accept a new ruleset to bind. Checks if it exists and complains, if not */ static rsRetVal setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) @@ -163,7 +182,6 @@ finalize_it: free(pszName); /* no longer needed */ RETiRet; } -#endif /* This function is a helper to runInput. I have extracted it @@ -181,8 +199,8 @@ finalize_it: * on scheduling order. -- rgerhards, 2008-10-02 */ static inline rsRetVal -processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, - uchar *fromHost, uchar *fromHostFQDN, uchar *fromHostIP) +processSocket(thrdInfo_t *pThrd, int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, + ruleset_t *pRuleset) { DEFiRet; int iNbrTimeUsed; @@ -196,8 +214,11 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, prop_t *propFromHostIP = NULL; char errStr[1024]; + assert(pThrd != NULL); iNbrTimeUsed = 0; while(1) { /* loop is terminated if we have a bad receive, done below in the body */ + if(pThrd->bShallStop == TRUE) + ABORT_FINALIZE(RS_RET_FORCE_TERM); socklen = sizeof(struct sockaddr_storage); lenRcvBuf = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if(lenRcvBuf < 0) { @@ -206,7 +227,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr); errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet"); } - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_ERR); // this most often is NOT an error, state is not checked by caller! } if(lenRcvBuf == 0) @@ -214,37 +235,39 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* if we reach this point, we had a good receive and can process the packet received */ /* check if we have a different sender than before, if so, we need to query some new values */ - if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { - CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); - memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ - /* Here we check if a host is permitted to send us - * syslog messages. If it isn't, we do not further - * process the message but log a warning (if we are - * configured to do this). - * rgerhards, 2005-09-26 - */ - *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", - (struct sockaddr *)&frominet, (char*)fromHostFQDN); - - if(!*pbIsPermitted) { - DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); - if(glbl.GetOption_DisallowWarning) { - time_t tt; - - time(&tt); - if(tt > ttLastDiscard + 60) { - ttLastDiscard = tt; - errmsg.LogError(0, NO_ERRCODE, - "UDP message from disallowed sender %s discarded", - (char*)fromHost); + if(bDoACLCheck) { + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us syslog messages. If it isn't, + * we do not further process the message but log a warning (if we are + * configured to do this). However, if the check would require name resolution, + * it is postponed to the main queue. See also my blog post at + * http://blog.gerhards.net/2009/11/acls-imudp-and-accepting-messages.html + * rgerhards, 2009-11-16 + */ + *pbIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)&frominet, "", 0); + + if(*pbIsPermitted == 0) { + DBGPRINTF("msg is not from an allowed sender\n"); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender discarded"); + } } } } + } else { + *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d)/%s,acl:%d,msg:%s\n", fd, (int) lenRcvBuf, fromHost, *pbIsPermitted, pRcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", fd, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf); - if(*pbIsPermitted) { + if(*pbIsPermitted != 0) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); } @@ -252,11 +275,12 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); MsgSetInputName(pMsg, pInputName); + MsgSetRuleset(pMsg, pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - pMsg->bParseHOSTNAME = 1; - MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); - CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; + if(*pbIsPermitted == 2) + pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */ + CHKiRet(msgSetFromSockinfo(pMsg, &frominet)); CHKiRet(submitMsg(pMsg)); } } @@ -271,45 +295,99 @@ finalize_it: } -/* This function is called to gather input. - * Note that udpLstnSocks must be non-NULL because otherwise we would not have - * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 - * rgerhards, 2008-10-07: I have implemented a very simple, yet in most cases probably - * highly efficient "name caching". Before querying a name, I now check if the name to be - * queried is the same as the one queried in the last message processed. If that is the - * case, we can simple re-use the previous value. This algorithm works quite well with - * few sender, especially if they emit messages in bursts. The more sender and the - * more intermixed messages arrive, the less this algorithm works, but the overhead - * is so minimal (a simple memory compare and move) that this does not hurt. Even - * with a real name lookup cache, this optimization here is useful as it is quicker - * than even a cache lookup). +/* This function implements the main reception loop. Depending on the environment, + * we either use the traditional (but slower) select() or the Linux-specific epoll() + * interface. ./configure settings control which one is used. + * rgerhards, 2009-09-09 */ -BEGINrunInput - int maxfds; +#if defined(HAVE_EPOLL_CREATE1) || defined(HAVE_EPOLL_CREATE) +#define NUM_EPOLL_EVENTS 10 +rsRetVal rcvMainLoop(thrdInfo_t *pThrd) +{ + DEFiRet; int nfds; + int efd; int i; struct sockaddr_storage frominetPrev; int bIsPermitted; - uchar fromHost[NI_MAXHOST]; - uchar fromHostIP[NI_MAXHOST]; - uchar fromHostFQDN[NI_MAXHOST]; -#ifdef USE_UNLIMITED_SELECT - fd_set *pReadfds = malloc(glbl.GetFdSetSize()); -#else - fd_set readfds; - fd_set *pReadfds = &readfds; -#endif + struct epoll_event *udpEPollEvt = NULL; + struct epoll_event currEvt[NUM_EPOLL_EVENTS]; + char errStr[1024]; -CODESTARTrunInput /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); - /* this is an endless loop - it is terminated when the thread is - * signalled to do so. This, however, is handled by the framework, - * right into the sleep below. + + CHKmalloc(udpEPollEvt = calloc(udpLstnSocks[0], sizeof(struct epoll_event))); + +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("imudp uses epoll_create1()\n"); + efd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("imudp uses epoll_create()\n"); + efd = epoll_create(NUM_EPOLL_EVENTS); +# endif + if(efd < 0) { + DBGPRINTF("epoll_create1() could not create fd\n"); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } + + /* fill the epoll set - we need to do this only once, as the set + * can not change dyamically. */ + for (i = 0; i < *udpLstnSocks; i++) { + if (udpLstnSocks[i+1] != -1) { + udpEPollEvt[i].events = EPOLLIN | EPOLLET; + udpEPollEvt[i].data.u64 = i+1; + if(epoll_ctl(efd, EPOLL_CTL_ADD, udpLstnSocks[i+1], &(udpEPollEvt[i])) < 0) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + errmsg.LogError(errno, NO_ERRCODE, "epoll_ctrl failed on fd %d with %s\n", + udpLstnSocks[i+1], errStr); + } + } + } + + while(1) { + /* wait for io to become ready */ + nfds = epoll_wait(efd, currEvt, NUM_EPOLL_EVENTS, -1); + DBGPRINTF("imudp: epoll_wait() returned with %d fds\n", nfds); + + if(pThrd->bShallStop == TRUE) + break; /* terminate input! */ + + for(i = 0 ; i < nfds ; ++i) { + processSocket(pThrd, udpLstnSocks[currEvt[i].data.u64], &frominetPrev, &bIsPermitted, + udpRulesets[currEvt[i].data.u64]); + } + } + +finalize_it: + if(udpEPollEvt != NULL) + free(udpEPollEvt); + + RETiRet; +} +#else /* #if HAVE_EPOLL_CREATE1 */ +/* this is the code for the select() interface */ +rsRetVal rcvMainLoop(thrdInfo_t *pThrd) +{ + DEFiRet; + int maxfds; + int nfds; + int i; + fd_set readfds; + struct sockaddr_storage frominetPrev; + int bIsPermitted; + + /* start "name caching" algo by making sure the previous system indicator + * is invalidated. + */ + bIsPermitted = 0; + memset(&frominetPrev, 0, sizeof(frominetPrev)); + DBGPRINTF("imudp uses select()\n"); + while(1) { /* Add the Unix Domain Sockets to the list of read * descriptors. @@ -318,40 +396,51 @@ CODESTARTrunInput * is given without -a, we do not need to listen at all.. */ maxfds = 0; - FD_ZERO (pReadfds); + FD_ZERO(&readfds); /* Add the UDP listen sockets to the list of read descriptors. */ for (i = 0; i < *udpLstnSocks; i++) { if (udpLstnSocks[i+1] != -1) { if(Debug) net.debugListenInfo(udpLstnSocks[i+1], "UDP"); - FD_SET(udpLstnSocks[i+1], pReadfds); + FD_SET(udpLstnSocks[i+1], &readfds); if(udpLstnSocks[i+1]>maxfds) maxfds=udpLstnSocks[i+1]; } } if(Debug) { dbgprintf("--------imUDP calling select, active file descriptors (max %d): ", maxfds); for (nfds = 0; nfds <= maxfds; ++nfds) - if ( FD_ISSET(nfds, pReadfds) ) + if(FD_ISSET(nfds, &readfds)) dbgprintf("%d ", nfds); dbgprintf("\n"); } /* wait for io to become ready */ - nfds = select(maxfds+1, (fd_set *) pReadfds, NULL, NULL, NULL); + nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ for(i = 0; nfds && i < *udpLstnSocks; i++) { - if(FD_ISSET(udpLstnSocks[i+1], pReadfds)) { - processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, - fromHost, fromHostFQDN, fromHostIP); + if(FD_ISSET(udpLstnSocks[i+1], &readfds)) { + processSocket(pThrd, udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, + udpRulesets[i+1]); --nfds; /* indicate we have processed one descriptor */ } } /* end of a run, back to loop for next recv() */ } - freeFdSet(pReadfds); - return iRet; + RETiRet; +} +#endif /* #if HAVE_EPOLL_CREATE1 */ + +/* This function is called to gather input. + * Note that udpLstnSocks must be non-NULL because otherwise we would not have + * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 + */ +BEGINrunInput +CODESTARTrunInput + iRet = rcvMainLoop(pThrd); ENDrunInput @@ -364,6 +453,7 @@ CODESTARTwillRun CHKiRet(prop.ConstructFinalize(pInputName)); net.PrintAllowedSenders(1); /* UDP */ + net.HasRestrictions(UCHAR_CONSTANT("UDP"), &bDoACLCheck); /* UDP */ /* if we could not set up any listners, there is no point in running... */ if(udpLstnSocks == NULL) @@ -371,9 +461,7 @@ CODESTARTwillRun iMaxLine = glbl.GetMaxLine(); - if((pRcvBuf = malloc((iMaxLine + 1) * sizeof(char))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pRcvBuf = MALLOC((iMaxLine + 1) * sizeof(char))); finalize_it: ENDwillRun @@ -385,6 +473,8 @@ CODESTARTafterRun if(udpLstnSocks != NULL) { net.closeUDPListenSockets(udpLstnSocks); udpLstnSocks = NULL; + free(udpRulesets); + udpRulesets = NULL; } if(pRcvBuf != NULL) { free(pRcvBuf); @@ -402,13 +492,22 @@ CODESTARTmodExit objRelease(glbl, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); objRelease(prop, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); ENDmodExit +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) @@ -417,10 +516,6 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a free(pszBindAddr); pszBindAddr = NULL; } - if(udpLstnSocks != NULL) { - net.closeUDPListenSockets(udpLstnSocks); - udpLstnSocks = NULL; - } iTimeRequery = TIME_REQUERY_DFLT;/* the default is to query only every second time */ return RS_RET_OK; } @@ -434,13 +529,12 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ - /* TODO: add - but this requires more changes, no time right now... - CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserverbindruleset", 0, eCmdHdlrGetWord, + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputudpserverbindruleset", 0, eCmdHdlrGetWord, setRuleset, NULL, STD_LOADABLE_MODULE_ID)); - */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserverrun", 0, eCmdHdlrGetWord, addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index daa3bb47..046f12f0 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-20 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -45,6 +45,7 @@ #include "glbl.h" #include "msg.h" #include "prop.h" +#include "debug.h" #include "unlimited_select.h" MODULE_TYPE_INPUT @@ -71,7 +72,8 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(prop) -static prop_t *pInputName = NULL; /* our inputName currently is always "imuxsock", and this will hold it */ +static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ +static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ static int startIndexUxLocalSockets; /* process funix from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup @@ -80,7 +82,7 @@ static int funixParseHost[MAXFUNIX] = { 0, }; /* should parser parse host name? static int funixFlags[MAXFUNIX] = { IGNDATE, }; /* should parser parse host name? read-only after startup */ static int funixCreateSockPath[MAXFUNIX] = { 0, }; /* auto-creation of socket directory? */ static uchar *funixn[MAXFUNIX] = { (uchar*) _PATH_LOG }; /* read-only after startup */ -static uchar *funixHName[MAXFUNIX] = { NULL, }; /* host-name override - if set, use this instead of actual name */ +static prop_t *funixHName[MAXFUNIX] = { NULL, }; /* host-name override - if set, use this instead of actual name */ static int funixFlowCtl[MAXFUNIX] = { eFLOWCTL_NO_DELAY, }; /* flow control settings for this socket */ static int funix[MAXFUNIX] = { -1, }; /* read-only after startup */ static int nfunix = 1; /* number of Unix sockets open / read-only after startup */ @@ -123,30 +125,41 @@ static rsRetVal setSystemLogFlowControl(void __attribute__((unused)) *pVal, int * rgerhards, 2007-12-20 * added capability to specify hostname for socket -- rgerhards, 2008-08-01 */ -static rsRetVal addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) +static rsRetVal +addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) { + DEFiRet; + if(nfunix < MAXFUNIX) { if(*pNewVal == ':') { funixParseHost[nfunix] = 1; - } - else { + } else { funixParseHost[nfunix] = 0; } - funixHName[nfunix] = pLogHostName; - pLogHostName = NULL; /* re-init for next, not freed because funixHName[] now owns it */ + CHKiRet(prop.Construct(&(funixHName[nfunix]))); + if(pLogHostName == NULL) { + CHKiRet(prop.SetString(funixHName[nfunix], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + } else { + CHKiRet(prop.SetString(funixHName[nfunix], pLogHostName, ustrlen(pLogHostName))); + /* reset hostname for next socket */ + free(pLogHostName); + pLogHostName = NULL; + } + CHKiRet(prop.ConstructFinalize(funixHName[nfunix])); funixFlowCtl[nfunix] = bUseFlowCtl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY; funixFlags[nfunix] = bIgnoreTimestamp ? IGNDATE : NOFLAG; funixCreateSockPath[nfunix] = bCreateSockPath; funixn[nfunix++] = pNewVal; - } - else { + } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", pNewVal); } - return RS_RET_OK; +finalize_it: + RETiRet; } + /* free the funixn[] socket names - needed as cleanup on several places * note that nfunix is NOT reset! funixn[0] is never freed, as it comes from * the constant memory pool - and if not, it is freeed via some other pointer. @@ -161,8 +174,7 @@ static rsRetVal discardFunixn(void) funixn[i] = NULL; } if(funixHName[i] != NULL) { - free(funixHName[i]); - funixHName[i] = NULL; + prop.Destruct(&(funixHName[i])); } } @@ -198,6 +210,35 @@ static int create_unix_socket(const char *path, int bCreatePath) } +/* submit received message to the queue engine + */ +static inline rsRetVal +SubmitMsg(uchar *pRcv, int lenRcv, int iSock) +{ + msg_t *pMsg; + DEFiRet; + + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstruct(&pMsg)); + MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); + MsgSetInputName(pMsg, pInputName); + MsgSetFlowControlType(pMsg, funixFlowCtl[iSock]); + + if(funixParseHost[iSock]) { + pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING | PARSE_HOSTNAME; + } else { + pMsg->msgFlags = funixFlags[iSock] | NEEDS_PARSING; + } + + MsgSetRcvFrom(pMsg, funixHName[iSock]); + CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); + CHKiRet(submitMsg(pMsg)); + +finalize_it: + RETiRet; +} + + /* This function receives data from a socket indicated to be ready * to receive and submits the message received for processing. * rgerhards, 2007-12-20 @@ -226,16 +267,13 @@ static rsRetVal readSocket(int fd, int iSock) if((size_t) iMaxLine < sizeof(bufRcv) - 1) { pRcv = bufRcv; } else { - CHKmalloc(pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))); + CHKmalloc(pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))); } iRcvd = recv(fd, pRcv, iMaxLine, 0); dbgprintf("Message from UNIX socket: #%d\n", fd); if (iRcvd > 0) { - parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], - (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock] ? (funixFlags[iSock] | PARSE_HOSTNAME) : funixFlags[iSock], - funixFlowCtl[iSock], pInputName, NULL, 0); + CHKiRet(SubmitMsg(pRcv, iRcvd, iSock)); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -296,8 +334,12 @@ CODESTARTrunInput /* wait for io to become ready */ nfds = select(maxfds+1, (fd_set *) pReadfds, NULL, NULL, NULL); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ for (i = 0; i < nfunix && nfds > 0; i++) { + if(glbl.GetGlobalInputTermState() == 1) + ABORT_FINALIZE(RS_RET_FORCE_TERM); /* terminate input! */ if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) { readSocket(fd, i); --nfds; /* indicate we have processed one */ @@ -305,6 +347,7 @@ CODESTARTrunInput } } +finalize_it: freeFdSet(pReadfds); RETiRet; ENDrunInput @@ -357,11 +400,8 @@ CODESTARTafterRun if (funixn[i] && funix[i] != -1) unlink((char*) funixn[i]); /* free no longer needed string */ - if(pLogSockName != NULL) - free(pLogSockName); - if(pLogHostName != NULL) { - free(pLogHostName); - } + free(pLogSockName); + free(pLogHostName); discardFunixn(); nfunix = 1; @@ -379,9 +419,17 @@ CODESTARTmodExit ENDmodExit +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURENonCancelInputTermination) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES ENDqueryEtryPt static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) @@ -423,6 +471,15 @@ CODEmodInit_QueryRegCFSLineHdlr funix[i] = -1; } + CHKiRet(prop.Construct(&pLocalHostIP)); + CHKiRet(prop.SetString(pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pLocalHostIP)); + + /* now init listen socket zero, the local log socket */ + CHKiRet(prop.Construct(&(funixHName[0]))); + CHKiRet(prop.SetString(funixHName[0], glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()))); + CHKiRet(prop.ConstructFinalize(funixHName[0])); + /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"omitlocallogging", 0, eCmdHdlrBinary, NULL, &bOmitLocalLogging, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/omdbalerting/Makefile.am b/plugins/omdbalerting/Makefile.am new file mode 100644 index 00000000..becf29b0 --- /dev/null +++ b/plugins/omdbalerting/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omdbalerting.la + +omdbalerting_la_SOURCES = omdbalerting.c +omdbalerting_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omdbalerting_la_LDFLAGS = -module -avoid-version +omdbalerting_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omdbalerting/omdbalerting.c b/plugins/omdbalerting/omdbalerting.c new file mode 100644 index 00000000..2e04391c --- /dev/null +++ b/plugins/omdbalerting/omdbalerting.c @@ -0,0 +1,144 @@ +/* omdbalerting.c + * generate alerts based on database contents - so far a skeleton + * left for implementation by somebody else (skeleton created on request). + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-11-17 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* config variables */ + + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omdbalerting:", sizeof(":dbalerting:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omdbalerting:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + /* we request the standard interface via template, others may be more useful + * here. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + // SAMPLE! CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomdbalertingensurelfending", 0, eCmdHdlrBinary, NULL, + // &bEnsureLFEnding, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 782ac22f..605e5ed9 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -193,7 +193,7 @@ static rsRetVal TCPSendGSSInit(void *pvData) base = (gss_base_service_name == NULL) ? "host" : gss_base_service_name; out_tok.length = strlen(pData->f_hname) + strlen(base) + 2; - CHKmalloc(out_tok.value = malloc(out_tok.length)); + CHKmalloc(out_tok.value = MALLOC(out_tok.length)); strcpy(out_tok.value, base); strcat(out_tok.value, "@"); strcat(out_tok.value, pData->f_hname); @@ -409,13 +409,13 @@ CODESTARTdoAction * hard-coded but this may be changed to a config parameter. * rgerhards, 2006-11-30 */ - if(pData->compressionLevel && (l > MIN_SIZE_FOR_COMPRESS)) { + if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { Bytef *out; uLongf destLen = sizeof(out) / sizeof(Bytef); uLong srcLen = l; int ret; /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ - CHKmalloc(out = (Bytef*) malloc(iMaxLine + iMaxLine/100 + 12)); + CHKmalloc(out = (Bytef*) MALLOC(iMaxLine + iMaxLine/100 + 12)); out[0] = 'z'; out[1] = '\0'; ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, @@ -563,7 +563,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) tmp = ++p; for(i=0 ; *p && isdigit((int) *p) ; ++p, ++i) /* SKIP AND COUNT */; - pData->port = malloc(i + 1); + pData->port = MALLOC(i + 1); if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store syslog forwarding port, " "using default port, results may not be what you intend\n"); diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index 3a7669c9..324e1a77 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -50,6 +50,7 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "datetime.h" #include "glbl.h" MODULE_TYPE_OUTPUT @@ -59,6 +60,7 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) +DEFobjCurrIf(datetime) /* we add a little support for multiple recipients. We do this via a * singly-linked list, enqueued from the top. -- rgerhards, 2008-08-04 @@ -478,7 +480,7 @@ mkSMTPTimestamp(uchar *pszBuf, size_t lenBuf) static const char szDay[][4] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; static const char szMonth[][4] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; - time(&tCurr); + datetime.GetTime(&tCurr); gmtime_r(&tCurr, &tmCurr); snprintf((char*)pszBuf, lenBuf, "Date: %s, %2d %s %4d %2d:%02d:%02d UT\r\n", szDay[tmCurr.tm_wday], tmCurr.tm_mday, szMonth[tmCurr.tm_mon], 1900 + tmCurr.tm_year, tmCurr.tm_hour, tmCurr.tm_min, tmCurr.tm_sec); @@ -669,6 +671,7 @@ CODESTARTmodExit freeConfigVariables(); /* release what we no longer need */ + objRelease(datetime, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -698,6 +701,7 @@ CODEmodInit_QueryRegCFSLineHdlr /* tell which objects we need */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); dbgprintf("ommail version %s initializing\n", VERSION); diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index eb774835..ffdcc532 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -65,6 +65,8 @@ typedef struct _instanceData { } instanceData; +static rsRetVal writePgSQL(uchar *psz, instanceData *pData); + BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance @@ -189,7 +191,8 @@ tryExec(uchar *pszCmd, instanceData *pData) * a sql format error - connection aborts were properly handled * before my patch. -- rgerhards, 2009-04-17 */ -rsRetVal writePgSQL(uchar *psz, instanceData *pData) +static rsRetVal +writePgSQL(uchar *psz, instanceData *pData) { int bHadError = 0; DEFiRet; @@ -227,16 +230,44 @@ BEGINtryResume CODESTARTtryResume if(pData->f_hpgsql == NULL) { iRet = initPgSQL(pData, 1); + if(iRet == RS_RET_OK) { + /* the code above seems not to actually connect to the database. As such, we do a + * dummy statement (a pointless select...) to verify the connection and return + * success only when that statemetn succeeds. Note that I am far from being a + * PostgreSQL expert, so any patch that does the desired result in a more + * intelligent way is highly welcome. -- rgerhards, 2009-12-16 + */ + iRet = writePgSQL((uchar*)"select 'a' as a", pData); + } + } ENDtryResume + +BEGINbeginTransaction +CODESTARTbeginTransaction +dbgprintf("ompgsql: beginTransaction\n"); + iRet = writePgSQL((uchar*) "begin", pData); /* TODO: make user-configurable */ +ENDbeginTransaction + + BEGINdoAction CODESTARTdoAction dbgprintf("\n"); - iRet = writePgSQL(ppString[0], pData); + CHKiRet(writePgSQL(ppString[0], pData)); + if(bCoreSupportsBatching) + iRet = RS_RET_DEFER_COMMIT; +finalize_it: ENDdoAction +BEGINendTransaction +CODESTARTendTransaction + iRet = writePgSQL((uchar*) "commit;", pData); /* TODO: make user-configurable */ +dbgprintf("ompgsql: endTransaction\n"); +ENDendTransaction + + BEGINparseSelectorAct int iPgSQLPropErr = 0; CODESTARTparseSelectorAct @@ -314,6 +345,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ ENDqueryEtryPt @@ -322,6 +354,9 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); + INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); + DBGPRINTF("ompgsql: module compiled with rsyslog version %s.\n", VERSION); + DBGPRINTF("ompgsql: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not "); ENDmodInit /* vi:set ai: */ diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index d5ef8b4f..349e45aa 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -43,6 +43,7 @@ #include "module-template.h" #include "glbl.h" #include "errmsg.h" +#include "debug.h" MODULE_TYPE_OUTPUT @@ -260,7 +261,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) tmp = ++p; for(i=0 ; *p && isdigit((int) *p) ; ++p, ++i) /* SKIP AND COUNT */; - pData->port = malloc(i + 1); + pData->port = MALLOC(i + 1); if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store relp port, " "using default port, results may not be what you intend\n"); diff --git a/plugins/omruleset/Makefile.am b/plugins/omruleset/Makefile.am new file mode 100644 index 00000000..fdd91a6e --- /dev/null +++ b/plugins/omruleset/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omruleset.la + +omruleset_la_SOURCES = omruleset.c +omruleset_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omruleset_la_LDFLAGS = -module -avoid-version +omruleset_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omruleset/omruleset.c b/plugins/omruleset/omruleset.c new file mode 100644 index 00000000..0e0fc13b --- /dev/null +++ b/plugins/omruleset/omruleset.c @@ -0,0 +1,231 @@ +/* omruleset.c + * This is a very special output module. It permits to pass a message object + * to another rule set. While this is a very simple action, it enables very + * complex configurations, e.g. it supports high-speed "and" conditions, sending + * data to the same file in a non-racy way, include functionality as well as + * some high-performance optimizations (in case the rule sets have the necessary + * queue definitions). So while this code is small, it is pretty important. + * + * NOTE: read comments in module-template.h for details on the calling interface! + * + * File begun on 2009-11-02 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <unistd.h> +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "ruleset.h" +#include "cfsysline.h" +#include "dirty.h" + +MODULE_TYPE_OUTPUT + +/* static data */ +DEFobjCurrIf(ruleset); +DEFobjCurrIf(errmsg); + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +/* config variables */ +ruleset_t *pRuleset = NULL; /* ruleset to enqueue message to (NULL = Default, not recommended) */ +uchar *pszRulesetName = NULL; + + +typedef struct _instanceData { + ruleset_t *pRuleset; /* ruleset to enqueue to */ + uchar *pszRulesetName; /* primarily for debugging/display purposes */ +} instanceData; + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance + free(pData->pszRulesetName); +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + dbgprintf("omruleset target %s[%p]\n", (char*) pData->pszRulesetName, pData->pRuleset); +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +/* Note that we change the flow control type to "no delay", because at this point in + * rsyslog procesing we can not really slow down the producer any longer, as we already + * work off a queue. So a delay would just block out execution for longer than needed. + */ +BEGINdoAction + msg_t *pMsg; +CODESTARTdoAction + CHKmalloc(pMsg = MsgDup((msg_t*) ppString[0])); + DBGPRINTF(":omruleset: forwarding message %p to ruleset %s[%p]\n", pMsg, + (char*) pData->pszRulesetName, pData->pRuleset); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + MsgSetRuleset(pMsg, pData->pRuleset); + submitMsg(pMsg); +finalize_it: +ENDdoAction + +/* set the ruleset name */ +static rsRetVal +setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, RS_RET_RULESET_NOT_FOUND, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pszRulesetName = pszName; /* save for later display purposes */ + +finalize_it: + if(iRet != RS_RET_OK) { /* cleanup needed? */ + free(pszName); + } + RETiRet; +} + + +BEGINparseSelectorAct + int iTplOpts; +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omruleset:", sizeof(":omruleset:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + if(pRuleset == NULL) { + errmsg.LogError(0, RS_RET_NO_RULESET, "error: no ruleset was specified, use " + "$ActionOmrulesetRulesetName directive first!"); + ABORT_FINALIZE(RS_RET_NO_RULESET); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omruleset:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + iTplOpts = OMSR_TPL_AS_MSG; + /* we call the message below because we need to call it via our interface definition. However, + * the format specified (if any) is always ignored. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, iTplOpts, (uchar*) "RSYSLOG_FileFormat")); + pData->pRuleset = pRuleset; + pData->pszRulesetName = pszRulesetName; + pRuleset = NULL; /* re-set, because there is a high risk of unwanted behavior if we leave it in! */ + pszRulesetName = NULL; /* note: we must not free, as we handed over this pointer to the instanceDat to the instanceDataa! */ +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit + free(pszRulesetName); + objRelease(errmsg, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + pRuleset = NULL; + RETiRet; +} + + +BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bMsgPassingSupported; /* does core support template passing as an array? */ +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + /* check if the rsyslog core supports parameter passing code */ + bMsgPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + /* found entry point, so let's see if core supports msg passing */ + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_MSG) + bMsgPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */ + } + + if(!bMsgPassingSupported) { + DBGPRINTF("omruleset: msg-passing is not supported by rsyslog core, can not continue.\n"); + ABORT_FINALIZE(RS_RET_NO_MSG_PASSING); + } + + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomrulesetrulesetname", 0, eCmdHdlrGetWord, + setRuleset, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/plugins/omsnmp/omsnmp.c b/plugins/omsnmp/omsnmp.c index 4db60e62..b973b09d 100644 --- a/plugins/omsnmp/omsnmp.c +++ b/plugins/omsnmp/omsnmp.c @@ -222,7 +222,7 @@ static rsRetVal omsnmp_sendsnmp(instanceData *pData, uchar *psz) ABORT_FINALIZE(RS_RET_DISABLE_ACTION); } - pdu->enterprise = (oid *) malloc(enterpriseoidlen * sizeof(oid)); + pdu->enterprise = (oid *) MALLOC(enterpriseoidlen * sizeof(oid)); memcpy(pdu->enterprise, enterpriseoid, enterpriseoidlen * sizeof(oid)); pdu->enterprise_length = enterpriseoidlen; diff --git a/plugins/omtesting/omtesting.c b/plugins/omtesting/omtesting.c index 411bcf88..9442f691 100644 --- a/plugins/omtesting/omtesting.c +++ b/plugins/omtesting/omtesting.c @@ -22,7 +22,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -46,12 +46,15 @@ #include <stdio.h> #include <stdarg.h> #include <stdlib.h> +#include <time.h> #include <string.h> #include <ctype.h> #include <assert.h> #include "dirty.h" #include "syslogd-types.h" #include "module-template.h" +#include "conf.h" +#include "cfsysline.h" MODULE_TYPE_OUTPUT @@ -59,9 +62,18 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA +static int bEchoStdout = 0; /* echo non-failed messages to stdout */ + typedef struct _instanceData { + enum { MD_SLEEP, MD_FAIL, MD_RANDFAIL, MD_ALWAYS_SUSPEND } + mode; + int bEchoStdout; int iWaitSeconds; int iWaitUSeconds; /* milli-seconds (one million of a second, just to make sure...) */ + int iCurrCallNbr; + int iFailFrequency; + int iResumeAfter; + int iCurrRetries; } instanceData; BEGINcreateInstance @@ -85,19 +97,106 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature -BEGINtryResume -CODESTARTtryResume -ENDtryResume +/* implement "fail" command in retry processing */ +static rsRetVal doFailOnResume(instanceData *pData) +{ + DEFiRet; -BEGINdoAction -CODESTARTdoAction + dbgprintf("fail retry curr %d, max %d\n", pData->iCurrRetries, pData->iResumeAfter); + if(++pData->iCurrRetries == pData->iResumeAfter) { + iRet = RS_RET_OK; + } else { + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +/* implement "fail" command */ +static rsRetVal doFail(instanceData *pData) +{ + DEFiRet; + + dbgprintf("fail curr %d, frquency %d\n", pData->iCurrCallNbr, pData->iFailFrequency); + if(pData->iCurrCallNbr++ % pData->iFailFrequency == 0) { + pData->iCurrRetries = 0; + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +/* implement "sleep" command */ +static rsRetVal doSleep(instanceData *pData) +{ + DEFiRet; struct timeval tvSelectTimeout; dbgprintf("sleep(%d, %d)\n", pData->iWaitSeconds, pData->iWaitUSeconds); tvSelectTimeout.tv_sec = pData->iWaitSeconds; tvSelectTimeout.tv_usec = pData->iWaitUSeconds; /* milli seconds */ select(0, NULL, NULL, NULL, &tvSelectTimeout); - //dbgprintf(":omtesting: end doAction(), iRet %d\n", iRet); + RETiRet; +} + + +/* implement "randomfail" command */ +static rsRetVal doRandFail(void) +{ + DEFiRet; + if((rand() >> 4) < (RAND_MAX >> 5)) { /* rougly same probability */ + iRet = RS_RET_OK; + dbgprintf("omtesting randfail: succeeded this time\n"); + } else { + iRet = RS_RET_SUSPENDED; + dbgprintf("omtesting randfail: failed this time\n"); + } + RETiRet; +} + + +BEGINtryResume +CODESTARTtryResume + dbgprintf("omtesting tryResume() called\n"); + switch(pData->mode) { + case MD_SLEEP: + break; + case MD_FAIL: + iRet = doFailOnResume(pData); + break; + case MD_RANDFAIL: + iRet = doRandFail(); + break; + case MD_ALWAYS_SUSPEND: + iRet = RS_RET_SUSPENDED; + } + dbgprintf("omtesting tryResume() returns iRet %d\n", iRet); +ENDtryResume + + +BEGINdoAction +CODESTARTdoAction + dbgprintf("omtesting received msg '%s'\n", ppString[0]); + switch(pData->mode) { + case MD_SLEEP: + iRet = doSleep(pData); + break; + case MD_FAIL: + iRet = doFail(pData); + break; + case MD_RANDFAIL: + iRet = doRandFail(); + case MD_ALWAYS_SUSPEND: + iRet = RS_RET_SUSPENDED; + } + + if(iRet == RS_RET_OK && pData->bEchoStdout) { + fprintf(stdout, "%s", ppString[0]); + fflush(stdout); + } + dbgprintf(":omtesting: end doAction(), iRet %d\n", iRet); ENDdoAction @@ -113,7 +212,7 @@ BEGINparseSelectorAct int i; uchar szBuf[1024]; CODESTARTparseSelectorAct -CODE_STD_STRING_REQUESTparseSelectorAct(0) +CODE_STD_STRING_REQUESTparseSelectorAct(1) /* code here is quick and dirty - if you like, clean it up. But keep * in mind it is just a testing aid ;) -- rgerhards, 2007-12-31 */ @@ -135,6 +234,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(0) if(isspace(*p)) ++p; + dbgprintf("omtesting command: '%s'\n", szBuf); if(!strcmp((char*) szBuf, "sleep")) { /* parse seconds */ for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { @@ -152,12 +252,43 @@ CODE_STD_STRING_REQUESTparseSelectorAct(0) if(isspace(*p)) ++p; pData->iWaitUSeconds = atoi((char*) szBuf); - } - /* once there are other modes, here is the spot to add it! */ - else { + pData->mode = MD_SLEEP; + } else if(!strcmp((char*) szBuf, "fail")) { + /* "fail fail-freqency resume-after" + * fail-frequency specifies how often doAction() fails + * resume-after speicifes how fast tryResume() should come back with success + * all numbers being "times called" + */ + /* parse fail-frequence */ + for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { + szBuf[i] = *p++; + } + szBuf[i] = '\0'; + if(isspace(*p)) + ++p; + pData->iFailFrequency = atoi((char*) szBuf); + /* parse resume-after */ + for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { + szBuf[i] = *p++; + } + szBuf[i] = '\0'; + if(isspace(*p)) + ++p; + pData->iResumeAfter = atoi((char*) szBuf); + pData->iCurrCallNbr = 1; + pData->mode = MD_FAIL; + } else if(!strcmp((char*) szBuf, "randfail")) { + pData->mode = MD_RANDFAIL; + } else if(!strcmp((char*) szBuf, "always_suspend")) { + pData->mode = MD_ALWAYS_SUSPEND; + } else { dbgprintf("invalid mode '%s', doing 'sleep 1 0' - fix your config\n", szBuf); } + pData->bEchoStdout = bEchoStdout; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (uchar*)"RSYSLOG_TraditionalForwardFormat")); + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -177,6 +308,10 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomtestingechostdout", 0, eCmdHdlrBinary, NULL, + &bEchoStdout, STD_LOADABLE_MODULE_ID)); + /* we seed the random-number generator in any case... */ + srand(time(NULL)); ENDmodInit /* * vi:set ai: diff --git a/plugins/omudpspoof/Makefile.am b/plugins/omudpspoof/Makefile.am new file mode 100644 index 00000000..79c495a0 --- /dev/null +++ b/plugins/omudpspoof/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omudpspoof.la + +omudpspoof_la_SOURCES = omudpspoof.c +omudpspoof_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(UDPSPOOF_CFLAGS) +omudpspoof_la_LDFLAGS = -module -avoid-version +omudpspoof_la_LIBADD = $(UDPSPOOF_LIBS) + +EXTRA_DIST = diff --git a/plugins/omudpspoof/omudpspoof.c b/plugins/omudpspoof/omudpspoof.c new file mode 100644 index 00000000..3ead5447 --- /dev/null +++ b/plugins/omudpspoof/omudpspoof.c @@ -0,0 +1,502 @@ +/* omudpspoof.c + * + * This is a udp-based output module that support spoofing. + * + * This file builds on UDP spoofing code contributed by + * David Lang <david@lang.hm>. I then created a "real" rsyslog module + * out of that code and omfwd. I decided to make it a separate module because + * omfwd already mixes up too many things (TCP & UDP & a differnt modes, + * this has historic reasons), it would not be a good idea to also add + * spoofing to it. And, looking at the requirements, there is little in + * common between omfwd and this module. + * + * Note: I have briefly checked libnet source code and I somewhat have the feeling + * that under some circumstances we may get into trouble with the lib. For + * example, it registers an atexit() handler, which should not play nicely + * with our dynamically loaded modules. Anyhow, I refrain from looking deeper + * at libnet code, especially as testing does not show any real issues. If some + * occur, it may be easier to modify libnet for dynamic load environments than + * using a work-around (as a side not, libnet looks somewhat unmaintained, the CVS + * I can see on sourceforge dates has no updates done less than 7 years ago). + * On the other hand, it looks like libnet is thread safe (at least is appropriately + * compiled, which I hope the standard packages are). So I do not guard calls to + * it with my own mutex calls. + * rgerhards, 2009-07-10 + * + * Copyright 2009 David Lang (spoofing code) + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <netinet/in.h> +#include <netdb.h> +#include <fnmatch.h> +#include <assert.h> +#include <errno.h> +#include <ctype.h> +#include <unistd.h> +#ifdef USE_NETZIP +#include <zlib.h> +#endif +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "net.h" +#include "template.h" +#include "msg.h" +#include "cfsysline.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "dirty.h" +#include "unicode-helper.h" +#include "debug.h" + + +#include <libnet.h> +#define _BSD_SOURCE 1 +#define __BSD_SOURCE 1 +#define __FAVOR_BSD 1 + + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(net) + +typedef struct _instanceData { + uchar *host; + uchar *port; + int *pSockArray; /* sockets to use for UDP */ + int compressionLevel; /* 0 - no compression, else level for zlib */ + struct addrinfo *f_addr; + u_short sourcePort; + u_short sourcePortStart; /* for sorce port iteration */ + u_short sourcePortEnd; +} instanceData; + +#define DFLT_SOURCE_PORT_START 32000 +#define DFLT_SOURCE_PORT_END 42000 + +/* config data */ +static uchar *pszTplName = NULL; /* name of the default template to use */ +static uchar *pszSourceNameTemplate = NULL; /* name of the template containing the spoofing address */ +static uchar *pszTargetHost = NULL; +static uchar *pszTargetPort = NULL; +static int iCompressionLevel = 0; /* zlib compressionlevel, the usual values */ +static int iSourcePortStart = DFLT_SOURCE_PORT_START; +static int iSourcePortEnd = DFLT_SOURCE_PORT_END; + + +/* add some variables needed for libnet */ +libnet_t *libnet_handle; +char errbuf[LIBNET_ERRBUF_SIZE]; + +/* forward definitions */ +static rsRetVal doTryResume(instanceData *pData); + + +/* Close the UDP sockets. + * rgerhards, 2009-05-29 + */ +static rsRetVal +closeUDPSockets(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + if(pData->pSockArray != NULL) { + net.closeUDPListenSockets(pData->pSockArray); + pData->pSockArray = NULL; + freeaddrinfo(pData->f_addr); + pData->f_addr = NULL; + } + RETiRet; +} + + +/* get the syslog forward port + * We may change the implementation to try to lookup the port + * if it is unspecified. So far, we use the IANA default auf 514. + * rgerhards, 2007-06-28 + */ +static inline uchar *getFwdPt(instanceData *pData) +{ + return (pData->port == NULL) ? UCHAR_CONSTANT("514") : pData->port; +} + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance + /* final cleanup */ + closeUDPSockets(pData); + free(pData->port); + free(pData->host); +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + DBGPRINTF("%s", pData->host); +ENDdbgPrintInstInfo + + +/* Send a message via UDP + * rgehards, 2007-12-20 + */ +static inline rsRetVal +UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) +{ + struct addrinfo *r; + int lsent = 0; + int bSendSuccess; + int j, build_ip; + u_char opt[20]; + struct sockaddr_in *tempaddr,source_ip; + libnet_ptag_t ip, ipo; + libnet_ptag_t udp; + DEFiRet; + + if(pData->pSockArray == NULL) { + CHKiRet(doTryResume(pData)); + } + + ip = ipo = udp = 0; + if(pData->sourcePort++ >= pData->sourcePortEnd){ + pData->sourcePort = pData->sourcePortStart; + } + + inet_pton(AF_INET, (char*)pszSourcename, &(source_ip.sin_addr)); + + bSendSuccess = FALSE; + for (r = pData->f_addr; r; r = r->ai_next) { + tempaddr = (struct sockaddr_in *)r->ai_addr; + libnet_clear_packet(libnet_handle); + /* note: libnet does need ports in host order NOT in network byte order! -- rgerhards, 2009-11-12 */ + udp = libnet_build_udp( + ntohs(pData->sourcePort),/* source port */ + ntohs(tempaddr->sin_port),/* destination port */ + LIBNET_UDP_H + len, /* packet length */ + 0, /* checksum */ + (u_char*)msg, /* payload */ + len, /* payload size */ + libnet_handle, /* libnet handle */ + udp); /* libnet id */ + if (udp == -1) { + DBGPRINTF("Can't build UDP header: %s\n", libnet_geterror(libnet_handle)); + } + + build_ip = 0; + /* this is not a legal options string */ + for (j = 0; j < 20; j++) { + opt[j] = libnet_get_prand(LIBNET_PR2); + } + ipo = libnet_build_ipv4_options(opt, 20, libnet_handle, ipo); + if (ipo == -1) { + DBGPRINTF("Can't build IP options: %s\n", libnet_geterror(libnet_handle)); + } + ip = libnet_build_ipv4( + LIBNET_IPV4_H + 20 + len + LIBNET_UDP_H, /* length */ + 0, /* TOS */ + 242, /* IP ID */ + 0, /* IP Frag */ + 64, /* TTL */ + IPPROTO_UDP, /* protocol */ + 0, /* checksum */ + source_ip.sin_addr.s_addr, + tempaddr->sin_addr.s_addr, + NULL, /* payload */ + 0, /* payload size */ + libnet_handle, /* libnet handle */ + ip); /* libnet id */ + if (ip == -1) { + DBGPRINTF("Can't build IP header: %s\n", libnet_geterror(libnet_handle)); + } + + /* Write it to the wire. */ + lsent = libnet_write(libnet_handle); + if (lsent == -1) { + DBGPRINTF("Write error: %s\n", libnet_geterror(libnet_handle)); + } else { + bSendSuccess = TRUE; + break; + } + } + /* finished looping */ + if (bSendSuccess == FALSE) { + DBGPRINTF("error forwarding via udp, suspending\n"); + iRet = RS_RET_SUSPENDED; + } + +finalize_it: + RETiRet; +} + + +/* try to resume connection if it is not ready + * rgerhards, 2007-08-02 + */ +static rsRetVal doTryResume(instanceData *pData) +{ + int iErr; + struct addrinfo *res; + struct addrinfo hints; + DEFiRet; + + if(pData->pSockArray != NULL) + FINALIZE; + + /* The remote address is not yet known and needs to be obtained */ + DBGPRINTF(" %s\n", pData->host); + memset(&hints, 0, sizeof(hints)); + /* port must be numeric, because config file syntax requires this */ + hints.ai_flags = AI_NUMERICSERV; + hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_socktype = SOCK_DGRAM; + if((iErr = (getaddrinfo((char*)pData->host, (char*)getFwdPt(pData), &hints, &res))) != 0) { + DBGPRINTF("could not get addrinfo for hostname '%s':'%s': %d%s\n", + pData->host, getFwdPt(pData), iErr, gai_strerror(iErr)); + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + DBGPRINTF("%s found, resuming.\n", pData->host); + pData->f_addr = res; + pData->pSockArray = net.create_udp_socket((uchar*)pData->host, NULL, 0); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pData->f_addr != NULL) { + freeaddrinfo(pData->f_addr); + pData->f_addr = NULL; + } + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +BEGINtryResume +CODESTARTtryResume + iRet = doTryResume(pData); +ENDtryResume + +BEGINdoAction + char *psz; /* temporary buffering */ + register unsigned l; + int iMaxLine; +CODESTARTdoAction + CHKiRet(doTryResume(pData)); + + iMaxLine = glbl.GetMaxLine(); + + DBGPRINTF(" %s:%s/udpspoofs\n", pData->host, getFwdPt(pData)); + + psz = (char*) ppString[0]; + l = strlen((char*) psz); + if((int) l > iMaxLine) + l = iMaxLine; + +# ifdef USE_NETZIP + /* Check if we should compress and, if so, do it. We also + * check if the message is large enough to justify compression. + * The smaller the message, the less likely is a gain in compression. + * To save CPU cycles, we do not try to compress very small messages. + * What "very small" means needs to be configured. Currently, it is + * hard-coded but this may be changed to a config parameter. + * rgerhards, 2006-11-30 + */ + if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { + Bytef *out; + uLongf destLen = iMaxLine + iMaxLine/100 +12; /* recommended value from zlib doc */ + uLong srcLen = l; + int ret; + /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ + CHKmalloc(out = (Bytef*) MALLOC(destLen)); + out[0] = 'z'; + out[1] = '\0'; + ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, + srcLen, pData->compressionLevel); + DBGPRINTF("Compressing message, length was %d now %d, return state %d.\n", + l, (int) destLen, ret); + if(ret != Z_OK) { + /* if we fail, we complain, but only in debug mode + * Otherwise, we are silent. In any case, we ignore the + * failed compression and just sent the uncompressed + * data, which is still valid. So this is probably the + * best course of action. + * rgerhards, 2006-11-30 + */ + DBGPRINTF("Compression failed, sending uncompressed message\n"); + } else if(destLen+1 < l) { + /* only use compression if there is a gain in using it! */ + DBGPRINTF("there is gain in compression, so we do it\n"); + psz = (char*) out; + l = destLen + 1; /* take care for the "z" at message start! */ + } + ++destLen; + } +# endif + + CHKiRet(UDPSend(pData, ppString[1], psz, l)); + +finalize_it: +ENDdoAction + + +BEGINparseSelectorAct + uchar *sourceTpl; +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(2) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omudpspoof:", sizeof(":omudpspoof:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omudpspoof:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + sourceTpl = (pszSourceNameTemplate == NULL) ? UCHAR_CONSTANT("RSYSLOG_omudpspoofDfltSourceTpl") + : pszSourceNameTemplate; + + if(pszTargetHost == NULL) { + errmsg.LogError(0, NO_ERRCODE, "No $ActionOMUDPSpoofTargetHost given, can not continue with this action."); + ABORT_FINALIZE(RS_RET_HOST_NOT_SPECIFIED); + } + + /* fill instance properties */ + CHKmalloc(pData->host = ustrdup(pszTargetHost)); + if(pszTargetPort == NULL) + pData->port = NULL; + else + CHKmalloc(pData->port = ustrdup(pszTargetPort)); + CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(sourceTpl), OMSR_NO_RQD_TPL_OPTS)); + pData->compressionLevel = iCompressionLevel; + pData->sourcePort = pData->sourcePortStart = iSourcePortStart; + pData->sourcePortEnd = iSourcePortEnd; + + /* process template */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_TraditionalForwardFormat" : pszTplName)); + +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +/* a common function to free our configuration variables - used both on exit + * and on $ResetConfig processing. -- rgerhards, 2008-05-16 + */ +static void +freeConfigVars(void) +{ + free(pszTplName); + pszTplName = NULL; + free(pszTargetHost); + pszTargetHost = NULL; + free(pszTargetPort); + pszTargetPort = NULL; +} + + +BEGINmodExit +CODESTARTmodExit + /* destroy the libnet state needed for forged UDP sources */ + libnet_destroy(libnet_handle); + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(net, LM_NET_FILENAME); + freeConfigVars(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +/* Reset config variables for this module to default values. + * rgerhards, 2008-03-28 + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + freeConfigVars(); + /* we now must reset all non-string values */ + iCompressionLevel = 0; + iSourcePortStart = DFLT_SOURCE_PORT_START; + iSourcePortEnd = DFLT_SOURCE_PORT_END; + return RS_RET_OK; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(net,LM_NET_FILENAME)); + + /* Initialize the libnet library. Root priviledges are required. + * this initializes a IPv4 socket to use for forging UDP packets. + */ + libnet_handle = libnet_init( + LIBNET_RAW4, /* injection type */ + NULL, /* network interface */ + errbuf); /* errbuf */ + + if(libnet_handle == NULL) { + errmsg.LogError(0, NO_ERRCODE, "Error initializing libnet, can not continue "); + ABORT_FINALIZE(RS_RET_ERR_LIBNET_INIT); + } + + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofdefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszTplName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofsourcenametemplate", 0, eCmdHdlrGetWord, NULL, &pszSourceNameTemplate, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspooftargethost", 0, eCmdHdlrGetWord, NULL, &pszTargetHost, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspooftargetport", 0, eCmdHdlrGetWord, NULL, &pszTargetPort, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofsourceportstart", 0, eCmdHdlrInt, NULL, &iSourcePortStart, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpspoofsourceportend", 0, eCmdHdlrInt, NULL, &iSourcePortEnd, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionomudpcompressionlevel", 0, eCmdHdlrInt, NULL, &iCompressionLevel, NULL)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vim:set ai: + */ diff --git a/plugins/pmlastmsg/Makefile.am b/plugins/pmlastmsg/Makefile.am new file mode 100644 index 00000000..f360243c --- /dev/null +++ b/plugins/pmlastmsg/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmlastmsg.la + +pmlastmsg_la_SOURCES = pmlastmsg.c +pmlastmsg_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmlastmsg_la_LDFLAGS = -module -avoid-version +pmlastmsg_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmlastmsg/pmlastmsg.c b/plugins/pmlastmsg/pmlastmsg.c new file mode 100644 index 00000000..275f1c1f --- /dev/null +++ b/plugins/pmlastmsg/pmlastmsg.c @@ -0,0 +1,175 @@ +/* pmlastmsg.c + * This is a parser module specifically for those horrible + * "<PRI>last message repeated n times" messages notoriously generated + * by some syslog implementations. Note that this parser should be placed + * on top of the parser stack -- it takes out only these messages and + * leaves all others for processing by the other parsers. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2010-07-13 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <ctype.h> +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.lastline") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +/* parse a legay-formatted syslog message. + */ +BEGINparse + uchar *p2parse; + int lenMsg; +#define OpeningText "last message repeated " +#define ClosingText " times" +CODESTARTparse + dbgprintf("Message will now be parsed by \"last message repated n times\" parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + + /* check if this message is of the type we handle in this (very limited) parser */ + /* first, we permit SP */ + while(lenMsg && *p2parse == ' ') { + --lenMsg; + ++p2parse; + } +dbgprintf("pmlastmsg: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < sizeof(OpeningText)-1 + sizeof(ClosingText)-1 + 1) { + /* too short, can not be "our" message */ +dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { + /* wrong opening text */ +dbgprintf("wrong opening text!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + lenMsg -= sizeof(OpeningText) - 1; + p2parse += sizeof(OpeningText) - 1; + + /* now we need an integer --> digits */ + while(lenMsg && isdigit(*p2parse)) { + --lenMsg; + ++p2parse; + } + + if(lenMsg != sizeof(ClosingText)-1) { + /* size must fit, else it is not "our" message... */ + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + if(strncasecmp((char*) p2parse, ClosingText, lenMsg) != 0) { + /* wrong closing text */ +dbgprintf("strcasecmp: %d\n", strncasecmp((char*) p2parse, ClosingText, lenMsg)); +dbgprintf("pmlastmsg: closing msg to look at: [%d]'%s', (%s)\n", lenMsg, p2parse, ClosingText); +dbgprintf("wrong closing text!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* OK, now we know we need to process this message, so we do that + * (and it is fairly simple in our case...) + */ + DBGPRINTF("pmlastmsg detected a \"last message repeated n times\" message\n"); + + setProtocolVersion(pMsg, 0); + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + MsgSetMSGoffs(pMsg, pMsg->offAfterPRI); /* we don't have a header! */ + MsgSetTAG(pMsg, (uchar*)"", 0); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ diff --git a/plugins/pmrfc3164sd/Makefile.am b/plugins/pmrfc3164sd/Makefile.am new file mode 100644 index 00000000..d1662d4d --- /dev/null +++ b/plugins/pmrfc3164sd/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmrfc3164sd.la + +pmrfc3164sd_la_SOURCES = pmrfc3164sd.c +pmrfc3164sd_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmrfc3164sd_la_LDFLAGS = -module -avoid-version +pmrfc3164sd_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmrfc3164sd/pmrfc3164sd.c b/plugins/pmrfc3164sd/pmrfc3164sd.c new file mode 100644 index 00000000..41a9252d --- /dev/null +++ b/plugins/pmrfc3164sd/pmrfc3164sd.c @@ -0,0 +1,341 @@ +/* pmrfc3164.c + * This is a parser module for RFC3164(legacy syslog)-formatted messages. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2009-11-04 by RGerhards + * + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include <ctype.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("contributed.rfc3164sd") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +/* Helper to parseRFCSyslogMsg. This function parses the structured + * data field of a message. It does NOT parse inside structured data, + * just gets the field as whole. Parsing the single entities is left + * to other functions. The parsepointer is advanced + * to after the terminating SP. The caller must ensure that the + * provided buffer is large enough to hold the to be extracted value. + * Returns 0 if everything is fine or 1 if either the field is not + * SP-terminated or any other error occurs. -- rger, 2005-11-24 + * The function now receives the size of the string and makes sure + * that it does not process more than that. The *pLenStr counter is + * updated on exit. -- rgerhards, 2009-09-23 + */ +static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr) +{ + uchar *p2parse; + int bCont = 1; + int iRet = 0; + int lenStr; + + assert(pp2parse != NULL); + assert(*pp2parse != NULL); + assert(pResult != NULL); + + p2parse = *pp2parse; + lenStr = *pLenStr; + + /* this is the actual parsing loop + * Remeber: structured data starts with [ and includes any characters + * until the first ] followed by a SP. There may be spaces inside + * structured data. There may also be \] inside the structured data, which + * do NOT terminate an element. + */ + + /* trim */ + while(lenStr > 0 && *p2parse == ' ') { + ++p2parse; /* eat SP, but only if not at end of string */ + --lenStr; + } + + if(lenStr == 0 || *p2parse != '[') + return 1; /* this is NOT structured data! */ + + if(*p2parse == '-') { /* empty structured data? */ + *pResult++ = '-'; + ++p2parse; + --lenStr; + } else { + while(bCont) { + if(lenStr < 2) { + /* we now need to check if we have only structured data */ + if(lenStr > 0 && *p2parse == ']') { + *pResult++ = *p2parse; + p2parse++; + lenStr--; + bCont = 0; + } else { + iRet = 1; /* this is not valid! */ + bCont = 0; + } + } else if(*p2parse == '\\' && *(p2parse+1) == ']') { + /* this is escaped, need to copy both */ + *pResult++ = *p2parse++; + *pResult++ = *p2parse++; + lenStr -= 2; + } else if(*p2parse == ']' && *(p2parse+1) == ' ') { + /* found end, just need to copy the ] and eat the SP */ + *pResult++ = *p2parse; + p2parse += 2; + lenStr -= 2; + bCont = 0; + } else { + *pResult++ = *p2parse++; + --lenStr; + } + } + } + + if(lenStr > 0 && *p2parse == ' ') { + ++p2parse; /* eat SP, but only if not at end of string */ + --lenStr; + } else { + iRet = 1; /* there MUST be an SP! */ + } + *pResult = '\0'; + + /* set the new parse pointer */ + *pp2parse = p2parse; + *pLenStr = lenStr; + return 0; +} + +/* parse a legay-formatted syslog message. + */ +BEGINparse + uchar *p2parse; + int lenMsg; + int bTAGCharDetected; + int i; /* general index for parsing */ + uchar bufParseTAG[CONF_TAG_MAXSIZE]; + uchar bufParseHOSTNAME[CONF_HOSTNAME_MAXSIZE]; + uchar *pBuf = NULL; +CODESTARTparse + dbgprintf("Message will now be parsed by the legacy syslog parser with structured-data support.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + setProtocolVersion(pMsg, 0); + + /* Check to see if msg contains a timestamp. We start by assuming + * that the message timestamp is the time of reception (which we + * generated ourselfs and then try to actually find one inside the + * message. There we go from high-to low precison and are done + * when we find a matching one. -- rgerhards, 2008-09-16 + */ + if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* we are done - parse pointer is moved by ParseTIMESTAMP3339 */; + } else if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; + } else if(*p2parse == ' ' && lenMsg > 1) { /* try to see if it is slighly malformed - HP procurve seems to do that sometimes */ + ++p2parse; /* move over space */ + --lenMsg; + if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* indeed, we got it! */ + /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; + } else {/* parse pointer needs to be restored, as we moved it off-by-one + * for this try. + */ + --p2parse; + ++lenMsg; + } + } + + if(pMsg->msgFlags & IGNDATE) { + /* we need to ignore the msg data, so simply copy over reception date */ + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + } + + /* rgerhards, 2006-03-13: next, we parse the hostname and tag. But we + * do this only when the user has not forbidden this. I now introduce some + * code that allows a user to configure rsyslogd to treat the rest of the + * message as MSG part completely. In this case, the hostname will be the + * machine that we received the message from and the tag will be empty. This + * is meant to be an interim solution, but for now it is in the code. + */ + if(bParseHOSTNAMEandTAG && !(pMsg->msgFlags & INTERNAL_MSG)) { + /* parse HOSTNAME - but only if this is network-received! + * rger, 2005-11-14: we still have a problem with BSD messages. These messages + * do NOT include a host name. In most cases, this leads to the TAG to be treated + * as hostname and the first word of the message as the TAG. Clearly, this is not + * of advantage ;) I think I have now found a way to handle this situation: there + * are certain characters which are frequently used in TAG (e.g. ':'), which are + * *invalid* in host names. So while parsing the hostname, I check for these characters. + * If I find them, I set a simple flag but continue. After parsing, I check the flag. + * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change + * the fields. I think this logic shall work with any type of syslog message. + * rgerhards, 2009-06-23: and I now have extended this logic to every character + * that is not a valid hostname. + */ + bTAGCharDetected = 0; + if(lenMsg > 0 && pMsg->msgFlags & PARSE_HOSTNAME) { + i = 0; + while(i < lenMsg && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.' + || p2parse[i] == '_' || p2parse[i] == '-') && i < (CONF_HOSTNAME_MAXSIZE - 1)) { + bufParseHOSTNAME[i] = p2parse[i]; + ++i; + } + + if(i == lenMsg) { + /* we have a message that is empty immediately after the hostname, + * but the hostname thus is valid! -- rgerhards, 2010-02-22 + */ + p2parse += i; + lenMsg -= i; + bufParseHOSTNAME[i] = '\0'; + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); + } else if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) { + /* we got a hostname! */ + p2parse += i + 1; /* "eat" it (including SP delimiter) */ + lenMsg -= i + 1; + bufParseHOSTNAME[i] = '\0'; + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); + } + } + + /* now parse TAG - that should be present in message from all sources. + * This code is somewhat not compliant with RFC 3164. As of 3164, + * the TAG field is ended by any non-alphanumeric character. In + * practice, however, the TAG often contains dashes and other things, + * which would end the TAG. So it is not desirable. As such, we only + * accept colon and SP to be terminators. Even there is a slight difference: + * a colon is PART of the TAG, while a SP is NOT part of the tag + * (it is CONTENT). Starting 2008-04-04, we have removed the 32 character + * size limit (from RFC3164) on the tag. This had bad effects on existing + * envrionments, as sysklogd didn't obey it either (probably another bug + * in RFC3164...). We now receive the full size, but will modify the + * outputs so that only 32 characters max are used by default. + */ + i = 0; + while(lenMsg > 0 && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *p2parse++; + --lenMsg; + } + if(lenMsg > 0 && *p2parse == ':') { + ++p2parse; + --lenMsg; + bufParseTAG[i++] = ':'; + } + + /* no TAG can only be detected if the message immediatly ends, in which case an empty TAG + * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23 + */ + bufParseTAG[i] = '\0'; /* terminate string */ + MsgSetTAG(pMsg, bufParseTAG, i); + } else {/* we enter this code area when the user has instructed rsyslog NOT + * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 + */ + if(!(pMsg->msgFlags & INTERNAL_MSG)) { + DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); + } + } + + CHKmalloc(pBuf = MALLOC(sizeof(uchar) * (lenMsg + 1))); + + /* STRUCTURED-DATA */ + parseRFCStructuredData(&p2parse, pBuf, &lenMsg); + MsgSetStructuredData(pMsg, (char*)pBuf); + + /* The rest is the actual MSG */ + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); + +finalize_it: + if(pBuf != NULL) + free(pBuf); +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("rfc3164sd parser init called\n"); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index c1a15198..f7db3e35 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -9,6 +9,7 @@ librsyslog_la_SOURCES = \ rsyslog.h \ unicode-helper.h \ atomic.h \ + batch.h \ syslogd-types.h \ module-template.h \ obj-types.h \ @@ -20,6 +21,8 @@ librsyslog_la_SOURCES = \ conf.h \ parser.h \ parser.c \ + strgen.h \ + strgen.c \ msg.c \ msg.h \ linkedlist.c \ @@ -136,7 +139,10 @@ lmnet_la_LDFLAGS = -module -avoid-version lmnet_la_LIBADD = # network stream master class and stream factory -lmnetstrms_la_SOURCES = netstrms.c netstrms.h netstrm.c netstrm.h nssel.c nssel.h +lmnetstrms_la_SOURCES = netstrms.c netstrms.h \ + netstrm.c netstrm.h \ + nssel.c nssel.h \ + nspoll.c nspoll.h lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version lmnetstrms_la_LIBADD = @@ -152,7 +158,9 @@ lmstrmsrv_la_LIBADD = # plain tcp driver - main driver pkglib_LTLIBRARIES += lmnsd_ptcp.la -lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h nsdsel_ptcp.c nsdsel_ptcp.h +lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h \ + nsdsel_ptcp.c nsdsel_ptcp.h \ + nsdpoll_ptcp.c nsdpoll_ptcp.h lmnsd_ptcp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnsd_ptcp_la_LDFLAGS = -module -avoid-version lmnsd_ptcp_la_LIBADD = diff --git a/runtime/apc.c b/runtime/apc.c index bc330e39..3c6b7ec4 100644 --- a/runtime/apc.c +++ b/runtime/apc.c @@ -40,9 +40,11 @@ #include "obj.h" #include "apc.h" #include "srUtils.h" +#include "datetime.h" /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(datetime) /* following is a used to implement a monotonically increasing id for the apcs. That * ID can be used to cancel an apc request. Note that the ID is generated with modulo @@ -200,7 +202,7 @@ unlistCurrent(apc_list_t **ppList) DEFiRet; assert(ppList != NULL); - time(&tCurr); + datetime.GetTime(&tCurr); if(apcListRoot == NULL || apcListRoot->pApc->ttExec > tCurr) { *ppList = NULL; @@ -249,12 +251,11 @@ execScheduled(void) apc_list_t *pExecList; apc_list_t *pCurr; apc_list_t *pNext; - DEFVARS_mutexProtection_uncond; DEFiRet; - BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + d_pthread_mutex_lock(&listMutex); iRet = unlistCurrent(&pExecList); - END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + d_pthread_mutex_unlock(&listMutex); CHKiRet(iRet); if(pExecList != NULL) { @@ -290,14 +291,12 @@ ENDobjConstruct(apc) static rsRetVal apcConstructFinalize(apc_t *pThis, apc_id_t *pID) { - DEFVARS_mutexProtection_uncond; DEFiRet; ISOBJ_TYPE_assert(pThis, apc); assert(pID != NULL); - BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + d_pthread_mutex_lock(&listMutex); insertApc(pThis, pID); - END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); -RUNLOG_STR("apcConstructFinalize post mutex unlock\n"); + d_pthread_mutex_unlock(&listMutex); RETiRet; } @@ -333,12 +332,10 @@ SetParam2(apc_t *pThis, void *param2) static rsRetVal CancelApc(apc_id_t id) { - DEFVARS_mutexProtection_uncond; - BEGINfunc - BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + d_pthread_mutex_lock(&listMutex); deleteApc(id); - END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + d_pthread_mutex_unlock(&listMutex); ENDfunc return RS_RET_OK; } @@ -380,7 +377,7 @@ ENDobjQueryInterface(apc) * rgerhards, 2009-04-06 */ BEGINObjClassExit(apc, OBJ_IS_CORE_MODULE) /* class, version */ - //objRelease(apcstk, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); pthread_mutex_destroy(&listMutex); ENDObjClassExit(apc) @@ -391,7 +388,7 @@ ENDObjClassExit(apc) */ BEGINObjClassInit(apc, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ - //CHKiRet(objUse(apcstk, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, apcDebugPrint); diff --git a/runtime/atomic.h b/runtime/atomic.h index fc3e0b2d..da544c4b 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -39,16 +39,18 @@ * They simply came in too late. -- rgerhards, 2008-04-02 */ #ifdef HAVE_ATOMIC_BUILTINS +# define ATOMIC_SUB(data, val, phlpmut) __sync_fetch_and_sub(data, val) +# define ATOMIC_ADD(data, val) __sync_fetch_and_add(&(data), val) # define ATOMIC_INC(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) -# define ATOMIC_INC_AND_FETCH(data) __sync_fetch_and_add(&(data), 1) +# define ATOMIC_INC_AND_FETCH(data, phlpmut) __sync_fetch_and_add(data, 1) # define ATOMIC_DEC(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) # define ATOMIC_DEC_AND_FETCH(data, phlpmut) __sync_sub_and_fetch(data, 1) -# define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) +# define ATOMIC_FETCH_32BIT(data, phlpmut) ((unsigned) __sync_fetch_and_and(data, 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) # define ATOMIC_STORE_0_TO_INT(data, phlpmut) __sync_fetch_and_and(data, 0) # define ATOMIC_STORE_1_TO_INT(data, phlpmut) __sync_fetch_and_or(data, 1) # define ATOMIC_STORE_INT_TO_INT(data, val) __sync_fetch_and_or(&(data), (val)) -# define ATOMIC_CAS(data, oldVal, newVal) __sync_bool_compare_and_swap(&(data), (oldVal), (newVal)); +# define ATOMIC_CAS(data, oldVal, newVal, phlpmut) __sync_bool_compare_and_swap(data, (oldVal), (newVal)) # define ATOMIC_CAS_VAL(data, oldVal, newVal, phlpmut) __sync_val_compare_and_swap(data, (oldVal), (newVal)); /* functions below are not needed if we have atomics */ @@ -77,18 +79,33 @@ } # define ATOMIC_STORE_0_TO_INT(data, hlpmut) { \ - pthread_mutex_lock(&hlpmut); \ + pthread_mutex_lock(hlpmut); \ *(data) = 0; \ - pthread_mutex_unlock(&hlpmut); \ + pthread_mutex_unlock(hlpmut); \ } # define ATOMIC_STORE_1_TO_INT(data, hlpmut) { \ - pthread_mutex_lock(&hlpmut); \ + pthread_mutex_lock(hlpmut); \ *(data) = 1; \ - pthread_mutex_unlock(&hlpmut); \ + pthread_mutex_unlock(hlpmut); \ } static inline int + ATOMIC_CAS(int *data, int oldVal, int newVal, pthread_mutex_t *phlpmut) { + int bSuccess; + pthread_mutex_lock(phlpmut); + if(*data == oldVal) { + *data = newVal; + bSuccess = 1; + } else { + bSuccess = 0; + } + pthread_mutex_unlock(phlpmut); + return(bSuccess); + } + + + static inline int ATOMIC_CAS_VAL(int *data, int oldVal, int newVal, pthread_mutex_t *phlpmut) { int val; pthread_mutex_lock(phlpmut); @@ -107,6 +124,15 @@ } static inline int + ATOMIC_INC_AND_FETCH(int *data, pthread_mutex_t *phlpmut) { + int val; + pthread_mutex_lock(phlpmut); + val = ++(*data); + pthread_mutex_unlock(phlpmut); + return(val); + } + + static inline int ATOMIC_DEC_AND_FETCH(int *data, pthread_mutex_t *phlpmut) { int val; pthread_mutex_lock(phlpmut); @@ -114,15 +140,30 @@ pthread_mutex_unlock(phlpmut); return(val); } + + static inline int + ATOMIC_FETCH_32BIT(int *data, pthread_mutex_t *phlpmut) { + int val; + pthread_mutex_lock(phlpmut); + val = (*data); + pthread_mutex_unlock(phlpmut); + return(val); + } + + static inline void + ATOMIC_SUB(int *data, int val, pthread_mutex_t *phlpmut) { + pthread_mutex_lock(phlpmut); + (*data) -= val; + pthread_mutex_unlock(phlpmut); + } #if 0 # warning "atomic builtins not available, using nul operations - rsyslogd will probably be racy!" # define ATOMIC_INC_AND_FETCH(data) (++(data)) -# define ATOMIC_FETCH_32BIT(data) (data) // TODO: del # define ATOMIC_STORE_1_TO_32BIT(data) (data) = 1 // TODO: del #endif # define DEF_ATOMIC_HELPER_MUT(x) pthread_mutex_t x # define INIT_ATOMIC_HELPER_MUT(x) pthread_mutex_init(&(x), NULL) -# define DESTROY_ATOMIC_HELPER_MUT(x) pthread_mutex_init(&(x), NULL) +# define DESTROY_ATOMIC_HELPER_MUT(x) pthread_mutex_destroy(&(x)) # define PREFER_ATOMIC_INC(data) ((void) ++data) diff --git a/runtime/batch.h b/runtime/batch.h new file mode 100644 index 00000000..68f48d8b --- /dev/null +++ b/runtime/batch.h @@ -0,0 +1,194 @@ +/* Definition of the batch_t data structure. + * I am not sure yet if this will become a full-blown object. For now, this header just + * includes the object definition and is not accompanied by code. + * + * Copyright 2009 by Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef BATCH_H_INCLUDED +#define BATCH_H_INCLUDED + +#include <string.h> +#include "msg.h" + +/* enum for batch states. Actually, we violate a layer here, in that we assume that a batch is used + * for action processing. So far, this seems acceptable, the status is simply ignored inside the + * main message queue. But over time, it could potentially be useful to split the two. + * rgerhad, 2009-05-12 + */ +typedef enum { + BATCH_STATE_RDY = 0, /* object ready for processing */ + BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ + BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unknown */ + BATCH_STATE_COMM = 3, /* message successfully commited */ + BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */ +} batch_state_t; + + +/* an object inside a batch, including any information (state!) needed for it to "life". + */ +struct batch_obj_s { + obj_t *pUsrp; /* pointer to user object (most often message) */ + batch_state_t state; /* associated state */ + /* work variables for action processing; these are reused for each action (or block of + * actions) + */ + sbool bFilterOK; /* work area for filter processing (per action, reused!) */ + sbool bPrevWasSuspended; + void *staticActParams[CONF_OMOD_NUMSTRINGS_MAXSIZE]; + /* a cache to save malloc(), if not absolutely necessary */ + size_t staticLenParams[CONF_OMOD_NUMSTRINGS_MAXSIZE]; + /* and the same for the message length (if used) */ + /* end action work variables */ +}; + +/* the batch + * This object is used to dequeue multiple user pointers which are than handed over + * to processing. The size of elements is fixed after queue creation, but may be + * modified by config variables (better said: queue properties). + * Note that a "user pointer" in rsyslog context so far always is a message + * object. We stick to the more generic term because queues may potentially hold + * other types of objects, too. + * rgerhards, 2009-05-12 + * Note that nElem is not necessarily equal to nElemDeq. This is the case when we + * discard some elements (because of configuration) during dequeue processing. As + * all Elements are only deleted when the batch is processed, we can not immediately + * delete them. So we need to keep their number that we can delete them when the batch + * is completed (else, the whole process does not work correctly). + */ +struct batch_s { + int maxElem; /* maximum number of elements that this batch supports */ + int nElem; /* actual number of element in this entry */ + int nElemDeq; /* actual number of elements dequeued (and thus to be deleted) - see comment above! */ + int iDoneUpTo; /* all messages below this index have state other than RDY */ + qDeqID deqID; /* ID of dequeue operation that generated this batch */ + int *pbShutdownImmediate;/* end processing of this batch immediately if set to 1 */ + sbool bSingleRuleset; /* do all msgs of this batch use a single ruleset? */ + batch_obj_t *pElem; /* batch elements */ +}; + + +/* some inline functions (we may move this off to an object .. or not) */ +static inline void +batchSetSingleRuleset(batch_t *pBatch, sbool val) { + pBatch->bSingleRuleset = val; +} + +/* get the batches ruleset (if we have a single ruleset) */ +static inline ruleset_t* +batchGetRuleset(batch_t *pBatch) { + return (pBatch->nElem > 0) ? ((msg_t*) pBatch->pElem[0].pUsrp)->pRuleset : NULL; +} + +/* get the ruleset of a specifc element of the batch (index not verified!) */ +static inline ruleset_t* +batchElemGetRuleset(batch_t *pBatch, int i) { + return ((msg_t*) pBatch->pElem[i].pUsrp)->pRuleset; +} + +/* get number of msgs for this batch */ +static inline int +batchNumMsgs(batch_t *pBatch) { + return pBatch->nElem; +} + + +/* set the status of the i-th batch element. Note that once the status is + * DISC, it will never be reset. So this function can NOT be used to initialize + * the state table. -- rgerhards, 2010-06-10 + */ +static inline void +batchSetElemState(batch_t *pBatch, int i, batch_state_t newState) { + if(pBatch->pElem[i].state != BATCH_STATE_DISC) + pBatch->pElem[i].state = newState; +} + + +/* check if an element is a valid entry. We do NOT verify if the + * element index is valid. -- rgerhards, 2010-06-10 + */ +static inline int +batchIsValidElem(batch_t *pBatch, int i) { + return(pBatch->pElem[i].bFilterOK && pBatch->pElem[i].state != BATCH_STATE_DISC); +} + + +/* copy one batch element to another. + * This creates a complete duplicate in those cases where + * it is needed. Use duplication only when absolutely necessary! + * rgerhards, 2010-06-10 + */ +static inline void +batchCopyElem(batch_obj_t *pDest, batch_obj_t *pSrc) { + memcpy(pDest, pSrc, sizeof(batch_obj_t)); +} + + +/* free members of a batch "object". Note that we can not do the usual + * destruction as the object typically is allocated on the stack and so the + * object itself cannot be freed! -- rgerhards, 2010-06-15 + */ +static inline void +batchFree(batch_t *pBatch) { + int i; + int j; + for(i = 0 ; i < pBatch->maxElem ; ++i) { + for(j = 0 ; j < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++j) { + free(pBatch->pElem[i].staticActParams[j]); + } + } + free(pBatch->pElem); +} + + +/* initialiaze a batch "object". The record must already exist, + * we "just" initialize it. The max number of elements must be + * provided. -- rgerhards, 2010-06-15 + */ +static inline rsRetVal +batchInit(batch_t *pBatch, int maxElem) { + DEFiRet; + pBatch->maxElem = maxElem; + CHKmalloc(pBatch->pElem = calloc((size_t)maxElem, sizeof(batch_obj_t))); + // TODO: replace calloc by inidividual writes? +finalize_it: + RETiRet; +} + + +/* primarily a helper for debug purposes, get human-readble name of state */ +static inline char * +batchState2String(batch_state_t state) { + switch(state) { + case BATCH_STATE_RDY: + return "BATCH_STATE_RDY"; + case BATCH_STATE_BAD: + return "BATCH_STATE_BAD"; + case BATCH_STATE_SUB: + return "BATCH_STATE_SUB"; + case BATCH_STATE_COMM: + return "BATCH_STATE_COMM"; + case BATCH_STATE_DISC: + return "BATCH_STATE_DISC"; + } + return "ERROR, batch state not known!"; +} +#endif /* #ifndef BATCH_H_INCLUDED */ diff --git a/runtime/conf.c b/runtime/conf.c index e2d3a889..d41f2950 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -93,19 +93,18 @@ DEFobjCurrIf(net) DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) -static int iNbrActions; /* number of actions the running config has. Needs to be init on ReInitConf() */ +static int iNbrActions = 0; /* number of currently defined actions */ -/* The following global variables are used for building +/* The following module-global variables are used for building * tag and host selector lines during startup and config reload. * This is stored as a global variable pool because of its ease. It is * also fairly compatible with multi-threading as the stratup code must - * be run in a single thread anyways. So there can be no race conditions. These - * variables are no longer used once the configuration has been loaded (except, - * of course, during a reload). rgerhards 2005-10-18 + * be run in a single thread anyways. So there can be no race conditions. + * rgerhards 2005-10-18 */ -EHostnameCmpMode eDfltHostnameCmpMode; -cstr_t *pDfltHostnameCmp; -cstr_t *pDfltProgNameCmp; +static EHostnameCmpMode eDfltHostnameCmpMode = HN_NO_COMP; +static cstr_t *pDfltHostnameCmp = NULL; +static cstr_t *pDfltProgNameCmp = NULL; /* process a directory and include all of its files into @@ -1041,7 +1040,7 @@ static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) DEFiRet; ASSERT(pp != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(f, rule); /* check which filter we need to pull... */ switch(**pp) { @@ -1063,6 +1062,7 @@ static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) * and, if so, we copy them over. rgerhards, 2005-10-18 */ if(pDfltProgNameCmp != NULL) { +RUNLOG_STR("dflt ProgNameCmp != NULL, setting opCSProgNameComp"); CHKiRet(rsCStrConstructFromCStr(&(f->pCSProgNameComp), pDfltProgNameCmp)); } @@ -1105,7 +1105,7 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); pAction->f_ReduceRepeated = 0; } - pAction->bEnabled = 1; /* action is enabled */ + pAction->eState = ACT_STATE_RDY; /* action is enabled */ iNbrActions++; /* one more active action! */ } break; @@ -1207,21 +1207,6 @@ cfline(uchar *line, rule_t **pfCurr) } -/* Reinitialize the configuration subsystem. This is a "work-around" to the fact - * that we do not yet have actual config objects. This method is to be called - * whenever a totally new config is started (which means on startup and HUP). - * Note that it MUST NOT be called for an included config file. - * rgerhards, 2008-07-28 - */ -static rsRetVal -ReInitConf(void) -{ - DEFiRet; - iNbrActions = 0; /* this is what we created the function for ;) - action count is reset */ - RETiRet; -} - - /* return the current number of active actions * rgerhards, 2008-07-28 */ @@ -1255,7 +1240,6 @@ CODESTARTobjQueryInterface(conf) pIf->doIncludeLine = doIncludeLine; pIf->cfline = cfline; pIf->processConfFile = processConfFile; - pIf->ReInitConf = ReInitConf; pIf->GetNbrActActions = GetNbrActActions; finalize_it: @@ -1267,6 +1251,15 @@ ENDobjQueryInterface(conf) */ BEGINObjClassExit(conf, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ CODESTARTObjClassExit(conf) + /* free no-longer needed module-global variables */ + if(pDfltHostnameCmp != NULL) { + rsCStrDestruct(&pDfltHostnameCmp); + } + + if(pDfltProgNameCmp != NULL) { + rsCStrDestruct(&pDfltProgNameCmp); + } + /* release objects we no longer need */ objRelease(expr, CORE_COMPONENT); objRelease(ctok, CORE_COMPONENT); diff --git a/runtime/conf.h b/runtime/conf.h index 25b887be..d85d1f82 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -37,20 +37,18 @@ BEGINinterface(conf) /* name must also be changed in ENDinterface macro! */ rsRetVal (*doIncludeLine)(uchar **pp, __attribute__((unused)) void* pVal); rsRetVal (*cfline)(uchar *line, rule_t **pfCurr); rsRetVal (*processConfFile)(uchar *pConfFile); - rsRetVal (*ReInitConf)(void); rsRetVal (*GetNbrActActions)(int *); ENDinterface(conf) -#define confCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define confCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ +/* in Version 3, entry point "ReInitConf()" was removed, as we do not longer need + * to support restart-type HUP -- rgerhards, 2009-07-15 + */ /* prototypes */ PROTOTYPEObj(conf); -/* TODO: remove them below (means move the config init code) -- rgerhards, 2008-02-19 */ -extern EHostnameCmpMode eDfltHostnameCmpMode; -extern cstr_t *pDfltHostnameCmp; -extern cstr_t *pDfltProgNameCmp; /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); diff --git a/runtime/datetime.c b/runtime/datetime.c index eff72f91..de26762d 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -127,6 +127,24 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) } +/* A fast alternative to getCurrTime() and time() that only obtains + * a timestamp like time() does. I was told that gettimeofday(), at + * least under Linux, is much faster than time() and I could confirm + * this testing. So I created that function as a replacement. + * rgerhards, 2009-11-12 + */ +static time_t +getTime(time_t *ttSeconds) +{ + struct timeval tp; + + if(gettimeofday(&tp, NULL) == -1) + return -1; + + if(ttSeconds != NULL) + *ttSeconds = tp.tv_sec; + return tp.tv_sec; +} /******************************************************************* @@ -842,6 +860,7 @@ CODESTARTobjQueryInterface(datetime) * of course, also affects the "if" above). */ pIf->getCurrTime = getCurrTime; + pIf->GetTime = getTime; pIf->ParseTIMESTAMP3339 = ParseTIMESTAMP3339; pIf->ParseTIMESTAMP3164 = ParseTIMESTAMP3164; pIf->formatTimestampToMySQL = formatTimestampToMySQL; diff --git a/runtime/datetime.h b/runtime/datetime.h index 9dcce3c5..82bd415b 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -41,8 +41,10 @@ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf); int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf, int); int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); + /* v3, 2009-11-12 */ + time_t (*GetTime)(time_t *ttSeconds); ENDinterface(datetime) -#define datetimeCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +#define datetimeCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ /* interface changes: * 1 - initial version * 2 - not compatible to 1 - bugfix required ParseTIMESTAMP3164 to accept char ** as @@ -50,6 +52,7 @@ ENDinterface(datetime) * third-party module should call. -- rgerhards, 2008.-09-12 * 3 - taken by v5 branch! * 4 - formatTimestamp3164 takes a third int parameter + * 5 - merge of versions 3 + 4 (2010-03-09) */ /* prototypes */ diff --git a/runtime/debug.c b/runtime/debug.c index 0ada909b..64e251e5 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -46,6 +46,9 @@ #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif #include "rsyslog.h" #include "debug.h" @@ -833,13 +836,12 @@ sigsegvHdlr(int signum) abort(); } -#if 1 -#pragma GCC diagnostic ignored "-Wempty-body" -/* write the debug message. This is a helper to dbgprintf and dbgoprint which - * contains common code. added 2008-09-26 rgerhards +/* actually write the debug message. This is a separate fuction because the cleanup_push/_pop + * interface otherwise is unsafe to use (generates compiler warnings at least). + * 2009-05-20 rgerhards */ -static void -dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +static inline void +do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg) { static pthread_t ptLastThrdID = 0; static int bWasNL = 0; @@ -847,22 +849,9 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) char pszWriteBuf[32*1024]; size_t lenWriteBuf; struct timespec t; - uchar *pszObjName = NULL; - int ret; - - /* we must get the object name before we lock the mutex, because the object - * potentially calls back into us. If we locked the mutex, we would deadlock - * ourselfs. On the other hand, the GetName call needs not to be protected, as - * this thread has a valid reference. If such an object is deleted by another - * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 - */ - if(pObj != NULL) { - pszObjName = obj.GetName(pObj); - } - - ret = pthread_mutex_lock(&mutdbgprint); - assert(ret == 0); /* make sure mutex operation does not fail */ - pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif /* The bWasNL handler does not really work. It works if no thread * switching occurs during non-NL messages. Else, things are messed @@ -888,7 +877,14 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(bWasNL) { if(bPrintTime) { +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%4.4ld.%9.9ld:", (long) (t.tv_sec % 10000), t.tv_nsec); if(stddbg != -1) write(stddbg, pszWriteBuf, lenWriteBuf); @@ -910,11 +906,35 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(altdbg != -1) write(altdbg, pszMsg, lenMsg); bWasNL = (pszMsg[lenMsg - 1] == '\n') ? 1 : 0; +} + +#pragma GCC diagnostic ignored "-Wempty-body" +/* write the debug message. This is a helper to dbgprintf and dbgoprint which + * contains common code. added 2008-09-26 rgerhards + */ +static void +dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +{ + uchar *pszObjName = NULL; + + /* we must get the object name before we lock the mutex, because the object + * potentially calls back into us. If we locked the mutex, we would deadlock + * ourselfs. On the other hand, the GetName call needs not to be protected, as + * this thread has a valid reference. If such an object is deleted by another + * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 + */ + if(pObj != NULL) { + pszObjName = obj.GetName(pObj); + } + + pthread_mutex_lock(&mutdbgprint); + pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); + + do_dbgprint(pszObjName, pszMsg, lenMsg); pthread_cleanup_pop(1); } #pragma GCC diagnostic warning "-Wempty-body" -#endif /* print some debug output when an object is given * This is mostly a copy of dbgprintf, but I do not know how to combine it @@ -1075,7 +1095,9 @@ int dbgEntrFunc(dbgFuncDB_t **ppFuncDB, const char *file, const char *func, int /* when we reach this point, we have a fully-initialized FuncDB! */ PREFER_ATOMIC_INC(pFuncDB->nTimesCalled); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) - dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + } if(pThrd->stackPtr >= (int) (sizeof(pThrd->callStack) / sizeof(dbgFuncDB_t*))) { dbgprintf("%s:%d: %s: debug module: call stack for this thread full, suspending call tracking\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); @@ -1105,10 +1127,12 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet) dbgFuncDBPrintActiveMutexes(pFuncDB, "WARNING: mutex still owned by us as we exit function, mutex: ", pthread_self()); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) { - if(iRet == RS_RET_NO_IRET) - dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); - else - dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + if(iRet == RS_RET_NO_IRET) + dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + else + dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + } } pThrd->stackPtr = iStackPtrRestore; if(pThrd->stackPtr < 0) { @@ -1254,6 +1278,20 @@ dbgPrintNameIsInList(const uchar *pName, dbgPrintName_t *pRoot) } +/* this is a special version of malloc that fills the alloced memory with + * HIGHVALUE, as this helps to identify bugs. -- rgerhards, 2009-10-22 + */ +void * +dbgmalloc(size_t size) +{ + void *pRet; + pRet = malloc(size); + if(pRet != NULL) + memset(pRet, 0xff, size); + return pRet; +} + + /* read in the runtime options * rgerhards, 2008-02-28 */ diff --git a/runtime/debug.h b/runtime/debug.h index cfdf819c..c011dd2d 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -3,7 +3,7 @@ * Definitions for the debug and run-time analysis support module. * Contains a lot of macros. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -104,6 +104,7 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet); void dbgSetExecLocation(int iStackPtr, int line); void dbgSetThrdName(uchar *pszName); void dbgPrintAllDebugInfo(void); +void *dbgmalloc(size_t size); /* macros */ #define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } @@ -131,6 +132,12 @@ void dbgPrintAllDebugInfo(void); # define RUNLOG_STR(str) #endif +#ifdef MEMCHECK +# define MALLOC(x) dbgmalloc(x) +#else +# define MALLOC(x) malloc(x) +#endif + /* mutex operations */ #define MUTOP_LOCKWAIT 1 #define MUTOP_LOCK 2 diff --git a/runtime/glbl.c b/runtime/glbl.c index 58558ed2..278bc4e1 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -39,6 +39,7 @@ #include "cfsysline.h" #include "glbl.h" #include "prop.h" +#include "atomic.h" /* some defaults */ #ifndef DFLT_NETSTRM_DRVR @@ -55,7 +56,7 @@ DEFobjCurrIf(prop) */ static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ -static int bHUPisRestart = 0; /* should SIGHUP cause a full system restart? */ +static int bParseHOSTNAMEandTAG = 1; /* parser modification (based on startup params!) */ static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ @@ -72,6 +73,10 @@ static uchar *pszDfltNetstrmDrvr = NULL; /* module name of default netstream dri static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm driver */ static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */ static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */ +static int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ +#ifndef HAVE_ATOMIC_BUILTINS +static DEF_ATOMIC_HELPER_MUT(mutTerminateInputs); +#endif #ifdef USE_UNLIMITED_SELECT static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */ #endif @@ -96,9 +101,9 @@ static dataType Get##nameFunc(void) \ return(nameVar); \ } +SIMP_PROP(ParseHOSTNAMEandTAG, bParseHOSTNAMEandTAG, int) SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) SIMP_PROP(PreserveFQDN, bPreserveFQDN, int) -SIMP_PROP(HUPisRestart, bHUPisRestart, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) @@ -123,6 +128,24 @@ SIMP_PROP_SET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) /* TO #undef SIMP_PROP_GET +/* return global input termination status + * rgerhards, 2009-07-20 + */ +static int GetGlobalInputTermState(void) +{ + return ATOMIC_FETCH_32BIT(&bTerminateInputs, &mutTerminateInputs); +} + + +/* set global termiantion state to "terminate". Note that this is a + * "once in a lifetime" action which can not be undone. -- gerhards, 2009-07-20 + */ +static void SetGlobalInputTermination(void) +{ + ATOMIC_STORE_1_TO_INT(&bTerminateInputs, &mutTerminateInputs); +} + + /* return our local hostname. if it is not set, "[localhost]" is returned */ static uchar* @@ -171,6 +194,7 @@ finalize_it: RETiRet; } + /* return our local hostname as a string property */ static prop_t* @@ -247,13 +271,15 @@ CODESTARTobjQueryInterface(glbl) pIf->GetWorkDir = GetWorkDir; pIf->GenerateLocalHostNameProperty = GenerateLocalHostNameProperty; pIf->GetLocalHostNameProp = GetLocalHostNameProp; + pIf->SetGlobalInputTermination = SetGlobalInputTermination; + pIf->GetGlobalInputTermState = GetGlobalInputTermState; #define SIMP_PROP(name) \ pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(ParseHOSTNAMEandTAG); SIMP_PROP(PreserveFQDN); - SIMP_PROP(HUPisRestart); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); @@ -302,7 +328,6 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; - bHUPisRestart = 0; bPreserveFQDN = 0; #ifdef USE_UNLIMITED_SELECT iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); @@ -328,9 +353,10 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); + + INIT_ATOMIC_HELPER_MUT(mutTerminateInputs); ENDObjClassInit(glbl) @@ -353,6 +379,7 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ if(LocalFQDNName != NULL) free(LocalFQDNName); objRelease(prop, CORE_COMPONENT); + DESTROY_ATOMIC_HELPER_MUT(mutTerminateInputs); ENDObjClassExit(glbl) /* vi:set ai: diff --git a/runtime/glbl.h b/runtime/glbl.h index 6a332576..4b4bdf83 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -8,7 +8,7 @@ * Please note that there currently is no glbl.c file as we do not yet * have any implementations. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -44,7 +44,6 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Set##name)(dataType); SIMP_PROP(MaxLine, int) SIMP_PROP(OptimizeUniProc, int) - SIMP_PROP(HUPisRestart, int) SIMP_PROP(PreserveFQDN, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) @@ -62,6 +61,11 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ /* added v3, 2009-06-30 */ rsRetVal (*GenerateLocalHostNameProperty)(void); prop_t* (*GetLocalHostNameProp)(void); + /* added v4, 2009-07-20 */ + int (*GetGlobalInputTermState)(void); + void (*SetGlobalInputTermination)(void); + /* added v5, 2009-11-03 */ + SIMP_PROP(ParseHOSTNAMEandTAG, int) /* note: v4, v5 are already used by more recent versions, so we need to skip them! */ /* added v6, 2009-11-16 as part of varmojfekoj's "unlimited select()" patch * Note that it must be always present, otherwise the interface would have different @@ -71,9 +75,11 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ * at some later stage). */ SIMP_PROP(FdSetSize, int) + /* v7: was neeeded to mean v5+v6 - do NOT add anything else for that version! */ + /* next change is v8! */ #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ diff --git a/runtime/module-template.h b/runtime/module-template.h index 3e963199..d05ec23c 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -46,6 +46,12 @@ DEFobjCurrIf(obj) #define DEF_LMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA +#define DEF_PMOD_STATIC_DATA \ + DEFobjCurrIf(obj) \ + DEF_MOD_STATIC_DATA +#define DEF_SMOD_STATIC_DATA \ + DEFobjCurrIf(obj) \ + DEF_MOD_STATIC_DATA /* Macro to define the module type. Each module can only have a single type. If @@ -65,6 +71,8 @@ static rsRetVal modGetType(eModType_t *modType) \ #define MODULE_TYPE_INPUT MODULE_TYPE(eMOD_IN) #define MODULE_TYPE_OUTPUT MODULE_TYPE(eMOD_OUT) +#define MODULE_TYPE_PARSER MODULE_TYPE(eMOD_PARSER) +#define MODULE_TYPE_STRGEN MODULE_TYPE(eMOD_STRGEN) #define MODULE_TYPE_LIB \ DEF_LMOD_STATIC_DATA \ MODULE_TYPE(eMOD_LIB) @@ -368,6 +376,17 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = endTransaction;\ } + +/* the following definition is a queryEtryPt block that must be added + * if a non-output module supports "isCompatibleWithFeature". + * rgerhards, 2009-07-20 + */ +#define CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "isCompatibleWithFeature")) {\ + *pEtryPoint = isCompatibleWithFeature;\ + } + + /* the following definition is the standard block for queryEtryPt for INPUT * modules. This can be used if no specific handling (e.g. to cover version * differences) is needed. @@ -389,6 +408,30 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ #define CODEqueryEtryPt_STD_LIB_QUERIES \ CODEqueryEtryPt_STD_MOD_QUERIES +/* the following definition is the standard block for queryEtryPt for PARSER + * modules. This can be used if no specific handling (e.g. to cover version + * differences) is needed. + */ +#define CODEqueryEtryPt_STD_PMOD_QUERIES \ + CODEqueryEtryPt_STD_MOD_QUERIES \ + else if(!strcmp((char*) name, "parse")) {\ + *pEtryPoint = parse;\ + } else if(!strcmp((char*) name, "GetParserName")) {\ + *pEtryPoint = GetParserName;\ + } + +/* the following definition is the standard block for queryEtryPt for Strgen + * modules. This can be used if no specific handling (e.g. to cover version + * differences) is needed. + */ +#define CODEqueryEtryPt_STD_SMOD_QUERIES \ + CODEqueryEtryPt_STD_MOD_QUERIES \ + else if(!strcmp((char*) name, "strgen")) {\ + *pEtryPoint = strgen;\ + } else if(!strcmp((char*) name, "GetName")) {\ + *pEtryPoint = GetStrgenName;\ + } + /* modInit() * This has an extra parameter, which is the specific name of the modInit * function. That is needed for built-in modules, which must have unique @@ -579,5 +622,58 @@ static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\ } +/* parse() - main entry point of parser modules + */ +#define BEGINparse \ +static rsRetVal parse(msg_t *pMsg)\ +{\ + DEFiRet; + +#define CODESTARTparse \ + assert(pMsg != NULL); + +#define ENDparse \ + RETiRet;\ +} + + +/* strgen() - main entry point of parser modules + */ +#define BEGINstrgen \ +static rsRetVal strgen(msg_t *pMsg, uchar **ppBuf, size_t *pLenBuf) \ +{\ + DEFiRet; + +#define CODESTARTstrgen \ + assert(pMsg != NULL); + +#define ENDstrgen \ + RETiRet;\ +} + + +/* function to specify the parser name. This is done via a single command which + * receives a ANSI string as parameter. + */ +#define PARSER_NAME(x) \ +static rsRetVal GetParserName(uchar **ppSz)\ +{\ + *ppSz = UCHAR_CONSTANT(x);\ + return RS_RET_OK;\ +} + + + +/* function to specify the strgen name. This is done via a single command which + * receives a ANSI string as parameter. + */ +#define STRGEN_NAME(x) \ +static rsRetVal GetStrgenName(uchar **ppSz)\ +{\ + *ppSz = UCHAR_CONSTANT(x);\ + return RS_RET_OK;\ +} + + /* vim:set ai: */ diff --git a/runtime/modules.c b/runtime/modules.c index 871f356a..d7362753 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -11,7 +11,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -57,10 +57,14 @@ #include "cfsysline.h" #include "modules.h" #include "errmsg.h" +#include "parser.h" +#include "strgen.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +DEFobjCurrIf(parser) +DEFobjCurrIf(strgen) /* we must ensure that only one thread at one time tries to load or unload * modules, otherwise we may see race conditions. This first came up with @@ -77,6 +81,26 @@ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ uchar *pModDir = NULL; /* read-only after startup */ +/* we provide a set of dummy functions for modules that do not support the + * some interfaces. + * On the commit feature: As the modules do not support it, they commit each message they + * receive, and as such the dummies can always return RS_RET_OK without causing + * harm. This simplifies things as in action processing we do not need to check + * if the transactional entry points exist. + */ +static rsRetVal dummyBeginTransaction() +{ + return RS_RET_OK; +} +static rsRetVal dummyEndTransaction() +{ + return RS_RET_OK; +} +static rsRetVal dummyIsCompatibleWithFeature() +{ + return RS_RET_INCOMPATIBLE; +} + #ifdef DEBUG /* we add some home-grown support to track our users (and detect who does not free us). In * the long term, this should probably be migrated into debug.c (TODO). -- rgerhards, 2008-03-11 @@ -216,19 +240,38 @@ static void moduleDestruct(modInfo_t *pThis) } +/* This enables a module to query the core for specific features. + * rgerhards, 2009-04-22 + */ +static rsRetVal queryCoreFeatureSupport(int *pBool, unsigned uFeat) +{ + DEFiRet; + + if((pBool == NULL)) + ABORT_FINALIZE(RS_RET_PARAM_ERROR); + + *pBool = (uFeat & CORE_FEATURE_BATCHING) ? 1 : 0; + +finalize_it: + RETiRet; +} + + /* The following function is the queryEntryPoint for host-based entry points. * Modules may call it to get access to core interface functions. Please note * that utility functions can be accessed via shared libraries - at least this * is my current shool of thinking. * Please note that the implementation as a query interface allows to take * care of plug-in interface version differences. -- rgerhards, 2007-07-31 + * ... but often it better not to use a new interface. So we now add core + * functions here that a plugin may request. -- rgerhards, 2009-04-22 */ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) { DEFiRet; if((name == NULL) || (pEtryPoint == NULL)) - return RS_RET_PARAM_ERROR; + ABORT_FINALIZE(RS_RET_PARAM_ERROR); if(!strcmp((char*) name, "regCfSysLineHdlr")) { *pEtryPoint = regCfSysLineHdlr; @@ -236,6 +279,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) *pEtryPoint = objGetObjInterface; } else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) { *pEtryPoint = OMSRgetSupportedTplOpts; + } else if(!strcmp((char*) name, "queryCoreFeatureSupport")) { + *pEtryPoint = queryCoreFeatureSupport; } else { *pEtryPoint = NULL; /* to be on the safe side */ ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND); @@ -361,10 +406,14 @@ finalize_it: static rsRetVal doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr) { - DEFiRet; rsRetVal localRet; modInfo_t *pNew = NULL; + uchar *pName; + parser_t *pParser; /* used for parser modules */ + strgen_t *pStrgen; /* used for strgen modules */ + rsRetVal (*GetName)(uchar**); rsRetVal (*modGetType)(eModType_t *pType); + DEFiRet; assert(modInit != NULL); @@ -392,6 +441,11 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ */ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modGetID", &pNew->modGetID)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modExit", &pNew->modExit)); + localRet = (*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->isCompatibleWithFeature = dummyIsCompatibleWithFeature; + else if(localRet != RS_RET_OK) + ABORT_FINALIZE(localRet); /* ... and now the module-specific interfaces */ switch(pNew->eType) { @@ -406,15 +460,70 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"dbgPrintInstInfo", &pNew->dbgPrintInstInfo)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"doAction", &pNew->mod.om.doAction)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct)); - CHKiRet((*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume)); /* try load optional interfaces */ localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) ABORT_FINALIZE(localRet); + + localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->mod.om.beginTransaction = dummyBeginTransaction; + else if(localRet != RS_RET_OK) + ABORT_FINALIZE(localRet); + + localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction); + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) { + pNew->mod.om.endTransaction = dummyEndTransaction; + } else if(localRet != RS_RET_OK) { + ABORT_FINALIZE(localRet); + } break; case eMOD_LIB: break; + case eMOD_PARSER: + /* first, we need to obtain the parser object. We could not do that during + * init as that would have caused class bootstrap issues which are not + * absolutely necessary. Note that we can call objUse() multiple times, it + * handles that. + */ + CHKiRet(objUse(parser, CORE_COMPONENT)); + /* here, we create a new parser object */ + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parse", &pNew->mod.pm.parse)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"GetParserName", &GetName)); + CHKiRet(GetName(&pName)); + CHKiRet(parser.Construct(&pParser)); + + /* check some features */ + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticSanitazion); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoSanitazion(pParser, TRUE)); + } + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticPRIParsing); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoPRIParsing(pParser, TRUE)); + } + + CHKiRet(parser.SetName(pParser, pName)); + CHKiRet(parser.SetModPtr(pParser, pNew)); + CHKiRet(parser.ConstructFinalize(pParser)); + break; + case eMOD_STRGEN: + /* first, we need to obtain the strgen object. We could not do that during + * init as that would have caused class bootstrap issues which are not + * absolutely necessary. Note that we can call objUse() multiple times, it + * handles that. + */ + CHKiRet(objUse(strgen, CORE_COMPONENT)); + /* here, we create a new parser object */ + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"strgen", &pNew->mod.sm.strgen)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"GetName", &GetName)); + CHKiRet(GetName(&pName)); + CHKiRet(strgen.Construct(&pStrgen)); + CHKiRet(strgen.SetName(pStrgen, pName)); + CHKiRet(strgen.SetModPtr(pStrgen, pNew)); + CHKiRet(strgen.ConstructFinalize(pStrgen)); + break; } pNew->pszName = (uchar*) strdup((char*)name); /* we do not care if strdup() fails, we can accept that */ @@ -461,14 +570,49 @@ static void modPrintList(void) case eMOD_LIB: dbgprintf("library"); break; + case eMOD_PARSER: + dbgprintf("parser"); + break; + case eMOD_STRGEN: + dbgprintf("strgen"); + break; } dbgprintf(" module.\n"); dbgprintf("Entry points:\n"); dbgprintf("\tqueryEtryPt: 0x%lx\n", (unsigned long) pMod->modQueryEtryPt); - dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); - dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); dbgprintf("\tdbgPrintInstInfo: 0x%lx\n", (unsigned long) pMod->dbgPrintInstInfo); dbgprintf("\tfreeInstance: 0x%lx\n", (unsigned long) pMod->freeInstance); + switch(pMod->eType) { + case eMOD_OUT: + dbgprintf("Output Module Entry Points:\n"); + dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction); + dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct); + dbgprintf("\ttryResume: 0x%lx\n", (unsigned long) pMod->tryResume); + dbgprintf("\tdoHUP: 0x%lx\n", (unsigned long) pMod->doHUP); + dbgprintf("\tBeginTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.beginTransaction == dummyBeginTransaction) ? + 0 : pMod->mod.om.beginTransaction)); + dbgprintf("\tEndTransaction: 0x%lx\n", (unsigned long) + ((pMod->mod.om.endTransaction == dummyEndTransaction) ? + 0 : pMod->mod.om.endTransaction)); + break; + case eMOD_IN: + dbgprintf("Input Module Entry Points\n"); + dbgprintf("\trunInput: 0x%lx\n", (unsigned long) pMod->mod.im.runInput); + dbgprintf("\twillRun: 0x%lx\n", (unsigned long) pMod->mod.im.willRun); + dbgprintf("\tafterRun: 0x%lx\n", (unsigned long) pMod->mod.im.afterRun); + break; + case eMOD_LIB: + break; + case eMOD_PARSER: + dbgprintf("Parser Module Entry Points\n"); + dbgprintf("\tparse: 0x%lx\n", (unsigned long) pMod->mod.pm.parse); + break; + case eMOD_STRGEN: + dbgprintf("Strgen Module Entry Points\n"); + dbgprintf("\tstrgen: 0x%lx\n", (unsigned long) pMod->mod.sm.strgen); + break; + } dbgprintf("\n"); pMod = GetNxt(pMod); /* done, go next */ } @@ -807,6 +951,7 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); /* We have a problem in our reference counting, which leads to this function * being called too early. This usually is no problem, but if we destroy * the mutex object, we get into trouble. So rather than finding the root cause, diff --git a/runtime/modules.h b/runtime/modules.h index 4d874019..49586e8d 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -12,7 +12,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -51,9 +51,11 @@ #define CURR_MOD_IF_VERSION 5 typedef enum eModType_ { - eMOD_IN, /* input module */ - eMOD_OUT, /* output module */ - eMOD_LIB /* library module - this module provides one or many interfaces */ + eMOD_IN = 0, /* input module */ + eMOD_OUT = 1, /* output module */ + eMOD_LIB = 2, /* library module */ + eMOD_PARSER = 3,/* parser module */ + eMOD_STRGEN = 4 /* strgen module */ } eModType_t; @@ -73,7 +75,7 @@ typedef enum eModLinkType_ { eMOD_LINK_ALL /* special: all linkage types, e.g. for unload */ } eModLinkType_t; -typedef struct modInfo_s { +struct modInfo_s { struct modInfo_s *pPrev; /* support for creating a double linked module list */ struct modInfo_s *pNext; /* support for creating a linked module list */ int iIFVers; /* Interface version of module */ @@ -111,11 +113,19 @@ typedef struct modInfo_s { struct {/* data for output modules */ /* below: perform the configured action */ + rsRetVal (*beginTransaction)(void*); rsRetVal (*doAction)(uchar**, unsigned, void*); + rsRetVal (*endTransaction)(void*); rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); } om; struct { /* data for library modules */ - } fm; + } lm; + struct { /* data for parser modules */ + rsRetVal (*parse)(msg_t*); + } pm; + struct { /* data for strgen modules */ + rsRetVal (*strgen)(msg_t*, uchar**, size_t *); + } sm; } mod; void *pModHdlr; /* handler to the dynamic library holding the module */ # ifdef DEBUG @@ -124,7 +134,8 @@ typedef struct modInfo_s { */ modUsr_t *pModUsrRoot; # endif -} modInfo_t; +}; + /* interfaces */ BEGINinterface(module) /* name must also be changed in ENDinterface macro! */ @@ -148,8 +159,4 @@ PROTOTYPEObj(module); /* TODO: remove them below (means move the config init code) -- rgerhards, 2008-02-19 */ extern uchar *pModDir; /* read-only after startup */ - #endif /* #ifndef MODULES_H_INCLUDED */ -/* - * vi:set ai: - */ diff --git a/runtime/msg.c b/runtime/msg.c index 57291def..1b188263 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -35,6 +35,8 @@ #include <string.h> #include <assert.h> #include <ctype.h> +#include <sys/socket.h> +#include <netdb.h> #if HAVE_MALLOC_H # include <malloc.h> #endif @@ -51,6 +53,7 @@ #include "unicode-helper.h" #include "ruleset.h" #include "prop.h" +#include "net.h" /* static data */ DEFobjStaticHelpers @@ -59,6 +62,7 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) DEFobjCurrIf(prop) +DEFobjCurrIf(net) static struct { uchar *pszName; @@ -274,8 +278,13 @@ static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warn static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23" }; +/* global variables */ +#if defined(HAVE_MALLOC_TRIM) && !defined(HAVE_ATOMIC_BUILTINS) +static pthread_mutex_t mutTrimCtr; /* mutex to handle malloc trim */ +#endif + /* some forward declarations */ -static int getAPPNAMELen(msg_t *pM, bool bLockMutex); +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); static inline int getProtocolVersion(msg_t *pM) @@ -284,11 +293,46 @@ static inline int getProtocolVersion(msg_t *pM) } +/* do a DNS reverse resolution, if not already done, reflect status + * rgerhards, 2009-11-16 + */ +static inline rsRetVal +resolveDNS(msg_t *pMsg) { + rsRetVal localRet; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + DEFiRet; + + CHKiRet(objUse(net, CORE_COMPONENT)); + if(pMsg->msgFlags & NEEDS_DNSRESOL) { + localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); + if(localRet == RS_RET_OK) { + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + } + } +finalize_it: + if(iRet != RS_RET_OK) { + /* best we can do: remove property */ + MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); + prop.Destruct(&propFromHost); + } + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; +} + + static inline void getInputName(msg_t *pM, uchar **ppsz, int *plen) { BEGINfunc - if(pM == NULL) { + if(pM == NULL || pM->pInputName == NULL) { *ppsz = UCHAR_CONSTANT(""); *plen = 0; } else { @@ -307,6 +351,7 @@ getRcvFromIP(msg_t *pM) if(pM == NULL) { psz = UCHAR_CONSTANT(""); } else { + resolveDNS(pM); /* make sure we have a resolved entry */ if(pM->pRcvFromIP == NULL) psz = UCHAR_CONSTANT(""); else @@ -626,13 +671,12 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) msg_t *pM; assert(ppThis != NULL); - CHKmalloc(pM = malloc(sizeof(msg_t))); + CHKmalloc(pM = MALLOC(sizeof(msg_t))); objConstructSetObjInfo(pM); /* intialize object helper entities */ /* initialize members in ORDER they appear in structure (think "cache line"!) */ pM->flowCtlType = 0; pM->bDoLock = 0; - pM->bParseHOSTNAME = 0; pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; @@ -661,7 +705,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pCSMSGID = NULL; pM->pInputName = NULL; pM->pRcvFromIP = NULL; - pM->pRcvFrom = NULL; + pM->rcvFrom.pRcvFrom = NULL; pM->pRuleset = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); @@ -745,6 +789,9 @@ static inline void freeHOSTNAME(msg_t *pThis) BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ int currRefCount; +# if HAVE_MALLOC_TRIM + int currCnt; +# endif CODESTARTobjDestruct(msg) /* DEV Debugging only ! dbgprintf("msgDestruct\t0x%lx, Ref now: %d\n", (unsigned long)pThis, pThis->iRefCount - 1); */ # ifdef HAVE_ATOMIC_BUILTINS @@ -762,8 +809,12 @@ CODESTARTobjDestruct(msg) freeHOSTNAME(pThis); if(pThis->pInputName != NULL) prop.Destruct(&pThis->pInputName); - if(pThis->pRcvFrom != NULL) - prop.Destruct(&pThis->pRcvFrom); + if((pThis->msgFlags & NEEDS_DNSRESOL) == 0) { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } else { + free(pThis->rcvFrom.pfrominet); + } if(pThis->pRcvFromIP != NULL) prop.Destruct(&pThis->pRcvFromIP); free(pThis->pszRcvdAt3164); @@ -800,18 +851,10 @@ CODESTARTobjDestruct(msg) * that we trim too often when the counter wraps. */ static unsigned iTrimCtr = 1; -# ifdef HAVE_ATOMICS - if(ATOMIC_INC_AND_FETCH(iTrimCtr) % 100000 == 0) { + currCnt = ATOMIC_INC_AND_FETCH(&iTrimCtr, &mutTrimCtr); + if(currCnt % 100000 == 0) { malloc_trim(128*1024); } -# else -static pthread_mutex_t mutTrimCtr = PTHREAD_MUTEX_INITIALIZER; - d_pthread_mutex_lock(&mutTrimCtr); - if(iTrimCtr++ % 100000 == 0) { - malloc_trim(128*1024); - } - d_pthread_mutex_unlock(&mutTrimCtr); -# endif } # endif } else { @@ -858,6 +901,7 @@ ENDobjDestruct(msg) msg_t* MsgDup(msg_t* pOld) { msg_t* pNew; + rsRetVal localRet; assert(pOld != NULL); @@ -870,7 +914,6 @@ msg_t* MsgDup(msg_t* pOld) pNew->iRefCount = 1; pNew->iSeverity = pOld->iSeverity; pNew->iFacility = pOld->iFacility; - pNew->bParseHOSTNAME = pOld->bParseHOSTNAME; pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; @@ -879,9 +922,19 @@ msg_t* MsgDup(msg_t* pOld) pNew->iLenMSG = pOld->iLenMSG; pNew->iLenTAG = pOld->iLenTAG; pNew->iLenHOSTNAME = pOld->iLenHOSTNAME; - if(pOld->pRcvFrom != NULL) { - pNew->pRcvFrom = pOld->pRcvFrom; - prop.AddRef(pNew->pRcvFrom); + if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) { + localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet); + if(localRet != RS_RET_OK) { + /* if something fails, we accept loss of this property, it is + * better than losing the whole message. + */ + pNew->msgFlags &= ~NEEDS_DNSRESOL; + } + } else { + if(pOld->rcvFrom.pRcvFrom != NULL) { + pNew->rcvFrom.pRcvFrom = pOld->rcvFrom.pRcvFrom; + prop.AddRef(pNew->rcvFrom.pRcvFrom); + } } if(pOld->pRcvFromIP != NULL) { pNew->pRcvFromIP = pOld->pRcvFromIP; @@ -944,7 +997,7 @@ msg_t* MsgDup(msg_t* pOld) * We do not serialize the cache properties. We re-create them when needed. * This saves us a lot of memory. Performance is no concern, as serializing * is a so slow operation that recration of the caches does not count. Also, - * we do not serialize bParseHOSTNAME, as this is only a helper variable + * we do not serialize --currently none--, as this is only a helper variable * during msg construction - and never again used later. * rgerhards, 2008-01-03 */ @@ -1145,15 +1198,21 @@ char *getProtocolVersionString(msg_t *pM) } -static char *getRawMsg(msg_t *pM) +static inline void +getRawMsg(msg_t *pM, uchar **pBuf, int *piLen) { - if(pM == NULL) - return ""; - else - if(pM->pszRawMsg == NULL) - return ""; - else - return (char*)pM->pszRawMsg; + if(pM == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + if(pM->pszRawMsg == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + *pBuf = pM->pszRawMsg; + *piLen = pM->iLenRawMsg; + } + } } @@ -1198,7 +1257,8 @@ static int getPRIi(msg_t *pM) /* Get PRI value in text form */ -static inline char *getPRI(msg_t *pM) +char * +getPRI(msg_t *pM) { /* PRI is a number in the range 0..191. Thus, we use a simple lookup table to obtain the * string value. It looks a bit clumpsy here in code ;) @@ -1213,7 +1273,8 @@ static inline char *getPRI(msg_t *pM) } -static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) +char * +getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -1234,7 +1295,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtMySQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_MySQL == NULL) { - if((pM->pszTIMESTAMP_MySQL = malloc(15)) == NULL) { + if((pM->pszTIMESTAMP_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } @@ -1245,7 +1306,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_PgSQL == NULL) { - if((pM->pszTIMESTAMP_PgSQL = malloc(21)) == NULL) { + if((pM->pszTIMESTAMP_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } @@ -1286,7 +1347,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtDefault: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } @@ -1297,7 +1358,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtMySQLDate: MsgLock(pM); if(pM->pszRcvdAt_MySQL == NULL) { - if((pM->pszRcvdAt_MySQL = malloc(15)) == NULL) { + if((pM->pszRcvdAt_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } @@ -1308,7 +1369,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszRcvdAt_PgSQL == NULL) { - if((pM->pszRcvdAt_PgSQL = malloc(21)) == NULL) { + if((pM->pszRcvdAt_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } @@ -1320,7 +1381,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtRFC3164BuggyDate: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } @@ -1332,7 +1393,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszRcvdAt3339 == NULL) { - if((pM->pszRcvdAt3339 = malloc(33)) == NULL) { + if((pM->pszRcvdAt3339 = MALLOC(33)) == NULL) { MsgUnlock(pM); return ""; } @@ -1497,7 +1558,7 @@ finalize_it: * This must be called WITHOUT the message lock being held. * rgerhards, 2009-06-26 */ -static inline void preparePROCID(msg_t *pM, bool bLockMutex) +static inline void preparePROCID(msg_t *pM, sbool bLockMutex) { if(pM->pCSPROCID == NULL) { if(bLockMutex == LOCK_MUTEX) @@ -1514,7 +1575,7 @@ static inline void preparePROCID(msg_t *pM, bool bLockMutex) #if 0 /* rgerhards, 2005-11-24 */ -static inline int getPROCIDLen(msg_t *pM, bool bLockMutex) +static inline int getPROCIDLen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); preparePROCID(pM, bLockMutex); @@ -1525,7 +1586,7 @@ static inline int getPROCIDLen(msg_t *pM, bool bLockMutex) /* rgerhards, 2005-11-24 */ -char *getPROCID(msg_t *pM, bool bLockMutex) +char *getPROCID(msg_t *pM, sbool bLockMutex) { ISOBJ_TYPE_assert(pM, msg); preparePROCID(pM, bLockMutex); @@ -1583,7 +1644,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) /* small enough: use fixed buffer (faster!) */ pBuf = pMsg->TAG.szBuf; } else { - if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { + if((pBuf = (uchar*) MALLOC(pMsg->iLenTAG + 1)) == NULL) { /* truncate message, better than completely loosing it... */ pBuf = pMsg->TAG.szBuf; pMsg->iLenTAG = CONF_TAG_BUFSIZE - 1; @@ -1604,7 +1665,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) * if there is a TAG and, if not, if it can emulate it. * rgerhards, 2005-11-24 */ -static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex) +static inline void tryEmulateTAG(msg_t *pM, sbool bLockMutex) { size_t lenTAG; uchar bufTAG[CONF_TAG_MAXSIZE]; @@ -1632,7 +1693,7 @@ static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex) } -static inline void +void getTAG(msg_t *pM, uchar **ppBuf, int *piLen) { if(pM == NULL) { @@ -1657,12 +1718,13 @@ int getHOSTNAMELen(msg_t *pM) if(pM == NULL) return 0; else - if(pM->pszHOSTNAME == NULL) - if(pM->pRcvFrom == NULL) + if(pM->pszHOSTNAME == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) return 0; else - return prop.GetStringLen(pM->pRcvFrom); - else + return prop.GetStringLen(pM->rcvFrom.pRcvFrom); + } else return pM->iLenHOSTNAME; } @@ -1673,12 +1735,13 @@ char *getHOSTNAME(msg_t *pM) return ""; else if(pM->pszHOSTNAME == NULL) { - if(pM->pRcvFrom == NULL) { + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) { return ""; } else { uchar *psz; int len; - prop.GetString(pM->pRcvFrom, &psz, &len); + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); return (char*) psz; } } else { @@ -1692,13 +1755,15 @@ uchar *getRcvFrom(msg_t *pM) uchar *psz; int len; BEGINfunc + if(pM == NULL) { psz = UCHAR_CONSTANT(""); } else { - if(pM->pRcvFrom == NULL) + resolveDNS(pM); + if(pM->rcvFrom.pRcvFrom == NULL) psz = UCHAR_CONSTANT(""); else - prop.GetString(pM->pRcvFrom, &psz, &len); + prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len); } ENDfunc return psz; @@ -1745,7 +1810,7 @@ static inline char *getStructuredData(msg_t *pM) /* check if we have a ProgramName, and, if not, try to aquire/emulate it. * rgerhards, 2009-06-26 */ -static inline void prepareProgramName(msg_t *pM, bool bLockMutex) +static inline void prepareProgramName(msg_t *pM, sbool bLockMutex) { if(pM->pCSProgName == NULL) { if(bLockMutex == LOCK_MUTEX) @@ -1764,7 +1829,7 @@ static inline void prepareProgramName(msg_t *pM, bool bLockMutex) /* get the length of the "programname" sz string * rgerhards, 2005-10-19 */ -int getProgramNameLen(msg_t *pM, bool bLockMutex) +int getProgramNameLen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); prepareProgramName(pM, bLockMutex); @@ -1775,10 +1840,10 @@ int getProgramNameLen(msg_t *pM, bool bLockMutex) /* get the "programname" as sz string * rgerhards, 2005-10-19 */ -char *getProgramName(msg_t *pM, bool bLockMutex) +uchar *getProgramName(msg_t *pM, sbool bLockMutex) { prepareProgramName(pM, bLockMutex); - return (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); + return (pM->pCSProgName == NULL) ? UCHAR_CONSTANT("") : rsCStrGetSzStrNoNULL(pM->pCSProgName); } @@ -1795,7 +1860,7 @@ static void tryEmulateAPPNAME(msg_t *pM) if(getProtocolVersion(pM) == 0) { /* only then it makes sense to emulate */ - MsgSetAPPNAME(pM, getProgramName(pM, MUTEX_ALREADY_LOCKED)); + MsgSetAPPNAME(pM, (char*)getProgramName(pM, MUTEX_ALREADY_LOCKED)); } } @@ -1805,7 +1870,7 @@ static void tryEmulateAPPNAME(msg_t *pM) * This must be called WITHOUT the message lock being held. * rgerhards, 2009-06-26 */ -static inline void prepareAPPNAME(msg_t *pM, bool bLockMutex) +static inline void prepareAPPNAME(msg_t *pM, sbool bLockMutex) { if(pM->pCSAPPNAME == NULL) { if(bLockMutex == LOCK_MUTEX) @@ -1822,7 +1887,7 @@ static inline void prepareAPPNAME(msg_t *pM, bool bLockMutex) /* rgerhards, 2005-11-24 */ -char *getAPPNAME(msg_t *pM, bool bLockMutex) +char *getAPPNAME(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); prepareAPPNAME(pM, bLockMutex); @@ -1831,7 +1896,7 @@ char *getAPPNAME(msg_t *pM, bool bLockMutex) /* rgerhards, 2005-11-24 */ -static int getAPPNAMELen(msg_t *pM, bool bLockMutex) +static int getAPPNAMELen(msg_t *pM, sbool bLockMutex) { assert(pM != NULL); prepareAPPNAME(pM, bLockMutex); @@ -1854,6 +1919,28 @@ void MsgSetInputName(msg_t *pThis, prop_t *inputName) } +/* Set the pfrominet socket store, so that we can obtain the peer at some + * later time. Note that we do not check if pRcvFrom is already set, so this + * function must only be called during message creation. + * NOTE: msgFlags is NOT set. While this is somewhat a violation of layers, + * it is done because it gains us some performance. So the caller must make + * sure the message flags are properly maintained. For all current callers, + * this is always the case and without extra effort required. + * rgerhards, 2009-11-17 + */ +rsRetVal +msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa){ + DEFiRet; + assert(pThis->rcvFrom.pRcvFrom == NULL); + + CHKmalloc(pThis->rcvFrom.pfrominet = malloc(sizeof(struct sockaddr_storage))); + memcpy(pThis->rcvFrom.pfrominet, sa, sizeof(struct sockaddr_storage)); + +finalize_it: + RETiRet; +} + + /* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef() * on the property, because this must be done in all current cases and there * is no case expected where this may not be necessary. @@ -1864,9 +1951,15 @@ void MsgSetRcvFrom(msg_t *pThis, prop_t *new) assert(pThis != NULL); prop.AddRef(new); - if(pThis->pRcvFrom != NULL) - prop.Destruct(&pThis->pRcvFrom); - pThis->pRcvFrom = new; + if(pThis->msgFlags & NEEDS_DNSRESOL) { + if(pThis->rcvFrom.pfrominet != NULL) + free(pThis->rcvFrom.pfrominet); + pThis->msgFlags &= ~NEEDS_DNSRESOL; + } else { + if(pThis->rcvFrom.pRcvFrom != NULL) + prop.Destruct(&pThis->rcvFrom.pRcvFrom); + } + pThis->rcvFrom.pRcvFrom = new; } @@ -1948,7 +2041,7 @@ void MsgSetHOSTNAME(msg_t *pThis, uchar* pszHOSTNAME, int lenHOSTNAME) if(pThis->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { /* small enough: use fixed buffer (faster!) */ pThis->pszHOSTNAME = pThis->szHOSTNAME; - } else if((pThis->pszHOSTNAME = (uchar*) malloc(pThis->iLenHOSTNAME + 1)) == NULL) { + } else if((pThis->pszHOSTNAME = (uchar*) MALLOC(pThis->iLenHOSTNAME + 1)) == NULL) { /* truncate message, better than completely loosing it... */ pThis->pszHOSTNAME = pThis->szHOSTNAME; pThis->iLenHOSTNAME = CONF_HOSTNAME_BUFSIZE - 1; @@ -2000,7 +2093,7 @@ rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG) lenNew = pThis->iLenRawMsg + lenMSG - pThis->iLenMSG; if(lenMSG > pThis->iLenMSG && lenNew >= CONF_RAWMSG_BUFSIZE) { /* we have lost our "bet" and need to alloc a new buffer ;) */ - CHKmalloc(bufNew = malloc(lenNew + 1)); + CHKmalloc(bufNew = MALLOC(lenNew + 1)); memcpy(bufNew, pThis->pszRawMsg, pThis->offMSG); if(pThis->pszRawMsg != pThis->szRawMsg) free(pThis->pszRawMsg); @@ -2033,7 +2126,7 @@ void MsgSetRawMsg(msg_t *pThis, char* pszRawMsg, size_t lenMsg) if(pThis->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { /* small enough: use fixed buffer (faster!) */ pThis->pszRawMsg = pThis->szRawMsg; - } else if((pThis->pszRawMsg = (uchar*) malloc(pThis->iLenRawMsg + 1)) == NULL) { + } else if((pThis->pszRawMsg = (uchar*) MALLOC(pThis->iLenRawMsg + 1)) == NULL) { /* truncate message, better than completely loosing it... */ pThis->pszRawMsg = pThis->szRawMsg; pThis->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; @@ -2089,7 +2182,7 @@ static uchar *getNOW(eNOWType eNow) uchar *pBuf; struct syslogTime t; - if((pBuf = (uchar*) malloc(sizeof(uchar) * tmpBUFSIZE)) == NULL) { + if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { return NULL; } @@ -2200,12 +2293,13 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, break; case PROP_HOSTNAME: pRes = (uchar*)getHOSTNAME(pMsg); + bufLen = getHOSTNAMELen(pMsg); break; case PROP_SYSLOGTAG: getTAG(pMsg, &pRes, &bufLen); break; case PROP_RAWMSG: - pRes = (uchar*)getRawMsg(pMsg); + getRawMsg(pMsg, &pRes, &bufLen); break; /* enable this, if someone actually uses UxTradMsg, delete after some time has * passed and nobody complained -- rgerhards, 2009-06-16 @@ -2226,7 +2320,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (uchar*)getPRI(pMsg); break; case PROP_PRI_TEXT: - pBuf = malloc(20 * sizeof(uchar)); + pBuf = MALLOC(20 * sizeof(uchar)); if(pBuf == NULL) { RET_OUT_OF_MEMORY; } else { @@ -2236,6 +2330,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, break; case PROP_IUT: pRes = UCHAR_CONSTANT("1"); /* always 1 for syslog messages (a MonitorWare thing;)) */ + bufLen = 1; break; case PROP_SYSLOGFACILITY: pRes = (uchar*)getFacility(pMsg); @@ -2253,7 +2348,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (uchar*)getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); break; case PROP_PROGRAMNAME: - pRes = (uchar*)getProgramName(pMsg, LOCK_MUTEX); + pRes = getProgramName(pMsg, LOCK_MUTEX); break; case PROP_PROTOCOL_VERSION: pRes = (uchar*)getProtocolVersionString(pMsg); @@ -2381,7 +2476,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* we got our end pointer, now do the copy */ /* TODO: code copied from below, this is a candidate for a separate function */ iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2427,7 +2522,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, ; /*DO NOTHING*/ } else { iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2547,7 +2642,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, iLenBuf = pmatch[pTpe->data.field.iSubMatchToUse].rm_eo - pmatch[pTpe->data.field.iSubMatchToUse].rm_so; - pB = malloc((iLenBuf + 1) * sizeof(uchar)); + pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if (pB == NULL) { if (*pbMustBeFreed == 1) @@ -2604,7 +2699,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, uchar *pBStart; uchar *pB; uchar *pSrc; - pBStart = pB = malloc((bufLen + 1) * sizeof(char)); + pBStart = pB = MALLOC((bufLen + 1) * sizeof(char)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2650,7 +2745,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2685,7 +2780,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else { if(bufLen == -1) bufLen = ustrlen(pRes); - pDst = pDstStart = malloc(bufLen + 1); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2724,7 +2819,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, int i; iLenBuf += iNumCC * 4; - pBStart = pB = malloc((iLenBuf + 1) * sizeof(uchar)); + pBStart = pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2769,7 +2864,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2804,7 +2899,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else { if(bufLen == -1) bufLen = ustrlen(pRes); - pDst = pDstStart = malloc(bufLen + 1); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2860,7 +2955,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* check if we need to obtain a private copy */ if(*pbMustBeFreed == 0) { /* ok, original copy, need a private one */ - pB = malloc((iLn + 1) * sizeof(uchar)); + pB = MALLOC((iLn + 1) * sizeof(uchar)); if(pB == NULL) { RET_OUT_OF_MEMORY; } @@ -2889,7 +2984,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, bufLen = ustrlen(pRes); iBufLen = bufLen; /* the malloc may be optimized, we currently use the worst case... */ - pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(uchar)); + pBStart = pDst = MALLOC((2 * iBufLen + 3) * sizeof(uchar)); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -3090,6 +3185,10 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) funcUnlock = MsgLockingDummy; funcDeleteMutex = MsgLockingDummy; funcMsgPrepareEnqueue = MsgLockingDummy; + /* some more inits */ +# if HAVE_MALLOC_TRIM + INIT_ATOMIC_HELPER_MUT(mutTrimCtr); +# endif ENDObjClassInit(msg) /* vim:set ai: */ diff --git a/runtime/msg.h b/runtime/msg.h index cda206fc..d42f1de2 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -60,15 +60,8 @@ struct msg { flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ pthread_mutex_t mut; - bool bDoLock; /* use the mutex? */ - bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ + sbool bDoLock; /* use the mutex? */ short iRefCount; /* reference counter (0 = unused) */ - /* background: the hostname is not present on "regular" messages - * received via UNIX domain sockets from the same machine. However, - * it is available when we have a forwarder (e.g. rfc3195d) using local - * sockets. All in all, the parser would need parse templates, that would - * resolve all these issues... rgerhards, 2005-10-06 - */ short iSeverity; /* the severity 0..7 */ short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ @@ -96,8 +89,12 @@ struct msg { cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ prop_t *pInputName; /* input name property */ - prop_t *pRcvFrom; /* name of system message was received from */ prop_t *pRcvFromIP; /* IP of system message was received from */ + union { + prop_t *pRcvFrom;/* name of system message was received from */ + struct sockaddr_storage *pfrominet; /* unresolved name */ + } rcvFrom; + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -115,8 +112,8 @@ struct msg { uchar *pszTAG; /* pointer to tag value */ uchar szBuf[CONF_TAG_BUFSIZE]; } TAG; - char pszTimestamp3164[16]; - char pszTimestamp3339[33]; + char pszTimestamp3164[CONST_LEN_TIMESTAMP_3164 + 1]; + char pszTimestamp3339[CONST_LEN_TIMESTAMP_3339 + 1]; char pszTIMESTAMP_SecFrac[7]; /* Note: a pointer is 64 bits/8 char, so this is actually fewer than a pointer! */ char pszRcvdAt_SecFrac[7]; /* same as above. Both are fractional seconds for their respective timestamp */ }; @@ -131,6 +128,8 @@ struct msg { #define MARK 0x008 /* this message is a mark */ #define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ #define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ +#define NEEDS_DNSRESOL 0x040 /* fromhost address is unresolved and must be locked up via DNS reverse lookup first */ +#define NEEDS_ACLCHK_U 0x080 /* check UDP ACLs after DNS resolution has been done in main queue consumer */ #define NO_PRI_IN_RAW 0x100 /* rawmsg does not include a PRI (Solaris!), but PRI is already set correctly in the msg object */ @@ -151,6 +150,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); +rsRetVal msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa); void MsgSetRcvFrom(msg_t *pMsg, prop_t*); void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); @@ -167,19 +167,22 @@ char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); uchar *getRcvFrom(msg_t *pM); +void getTAG(msg_t *pM, uchar **ppBuf, int *piLen); +char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt); +char *getPRI(msg_t *pMsg); /* TODO: remove these five (so far used in action.c) */ uchar *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); -char *getPROCID(msg_t *pM, bool bLockMutex); -char *getAPPNAME(msg_t *pM, bool bLockMutex); +char *getPROCID(msg_t *pM, sbool bLockMutex); +char *getAPPNAME(msg_t *pM, sbool bLockMutex); int getMSGLen(msg_t *pM); char *getHOSTNAME(msg_t *pM); int getHOSTNAMELen(msg_t *pM); -char *getProgramName(msg_t *pM, bool bLockMutex); -int getProgramNameLen(msg_t *pM, bool bLockMutex); +uchar *getProgramName(msg_t *pM, sbool bLockMutex); +int getProgramNameLen(msg_t *pM, sbool bLockMutex); uchar *getRcvFrom(msg_t *pM); rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); uchar *propIDToName(propid_t propID); @@ -211,6 +214,16 @@ MsgSetRawMsgSize(msg_t *pMsg, size_t newLen) } +/* get the ruleset that is associated with the ruleset. + * May be NULL. -- rgerhards, 2009-10-27 + */ +static inline ruleset_t* +MsgGetRuleset(msg_t *pMsg) +{ + return pMsg->pRuleset; +} + + #endif /* #ifndef MSG_H_INCLUDED */ /* vim:set ai: */ diff --git a/runtime/net.c b/runtime/net.c index 4eb5a3bb..7653ea1d 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -173,7 +173,7 @@ AddPermittedPeerWildcard(permittedPeers_t *pPeer, uchar* pszStr, size_t lenStr) /* alloc memory for the domain component. We may waste a byte or * two, but that's ok. */ - CHKmalloc(pNew->pszDomainPart = malloc(lenStr +1 )); + CHKmalloc(pNew->pszDomainPart = MALLOC(lenStr +1 )); } if(pszStr[0] == '*') { @@ -695,7 +695,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS case AF_INET: /* add IPv4 */ iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -710,7 +710,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(sizeof(struct sockaddr_in))) + if((allowIP.addr.NetAddr = MALLOC(sizeof(struct sockaddr_in))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -732,7 +732,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 128; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -892,15 +892,18 @@ rsRetVal addAllowedSenderLine(char* pName, uchar** ppRestOfConfLine) * including IPv4/v6 as well as domain name wildcards. * This is a helper to isAllowedSender. As it is only called once, it is * declared inline. - * Returns 0 if they do not match, something else otherwise. - * contributed 1007-07-16 by mildew@gmail.com + * Returns 0 if they do not match, 1 if they match and 2 if a DNS name would have been required. + * contributed 2007-07-16 by mildew@gmail.com */ -static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost) +static inline int +MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { assert(pAllow != NULL); assert(pFrom != NULL); if(F_ISSET(pAllow->flags, ADDR_NAME)) { + if(bChkDNS == 0) + return 2; dbgprintf("MaskCmp: host=\"%s\"; pattern=\"%s\"\n", pszFromHost, pAllow->addr.HostWildcard); # if !defined(FNM_CASEFOLD) @@ -967,18 +970,22 @@ static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr /* check if a sender is allowed. The root of the the allowed sender. * list must be proveded by the caller. As such, this function can be * used to check both UDP and TCP allowed sender lists. - * returns 1, if the sender is allowed, 0 otherwise. + * returns 1, if the sender is allowed, 0 if not and 2 if we could not + * obtain a result because we would need a dns name, which we don't have + * (2 was added rgerhards, 2009-11-16). * rgerhards, 2005-09-26 */ -static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) +static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS) { struct AllowedSenders *pAllow; struct AllowedSenders *pAllowRoot; + int bNeededDNS = 0; /* partial check because we could not resolve DNS? */ + int ret; assert(pFrom != NULL); if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK) - return 0; /* if something went wrong, we denie access - that's the better choice... */ + return 0; /* if something went wrong, we deny access - that's the better choice... */ if(pAllowRoot == NULL) return 1; /* checking disabled, everything is valid! */ @@ -990,10 +997,20 @@ static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *p * that the sender is disallowed. */ for(pAllow = pAllowRoot ; pAllow != NULL ; pAllow = pAllow->pNext) { - if (MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost)) + ret = MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost, bChkDNS); + if(ret == 1) return 1; + else if(ret == 2) + bNeededDNS = 2; } - return 0; + return bNeededDNS; +} + + +/* legacy API, not to be used any longer */ +static int +isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) { + return isAllowedSender2(pszType, pFrom, pszFromHost, 1); } @@ -1306,7 +1323,7 @@ getLocalHostname(uchar **ppName) do { if(buf == NULL) { buf_len = 128; /* Initial guess */ - CHKmalloc(buf = malloc(buf_len)); + CHKmalloc(buf = MALLOC(buf_len)); } else { buf_len += buf_len; CHKmalloc(buf = realloc (buf, buf_len)); @@ -1370,7 +1387,7 @@ int *create_udp_socket(uchar *hostname, uchar *pszPort, int bIsServer) /* Count max number of sockets we may open */ for (maxs = 0, r = res; r != NULL ; r = r->ai_next, maxs++) /* EMPTY */; - socks = malloc((maxs+1) * sizeof(int)); + socks = MALLOC((maxs+1) * sizeof(int)); if (socks == NULL) { errmsg.LogError(0, NO_ERRCODE, "couldn't allocate memory for UDP sockets, suspending UDP message reception"); freeaddrinfo(res); @@ -1533,12 +1550,36 @@ static int CmpHost(struct sockaddr_storage *s1, struct sockaddr_storage* s2, siz ret = memcmp(s1, s2, socklen); } -dbgprintf("CmpHost returns %d\n", ret); finalize_it: return ret; } + +/* check if restrictions (ALCs) exists. The goal of this function is to disable the + * somewhat time-consuming ACL checks if no restrictions are defined (the usual case). + * This also permits to gain some speedup by using firewall-based ACLs instead of + * rsyslog ACLs (the recommended method. + * rgerhards, 2009-11-16 + */ +static rsRetVal +HasRestrictions(uchar *pszType, int *bHasRestrictions) { + struct AllowedSenders *pAllowRoot; + DEFiRet; + + CHKiRet(setAllowRoot(&pAllowRoot, pszType)); + + *bHasRestrictions = (pAllowRoot == NULL) ? 0 : 1; + +finalize_it: + if(iRet != RS_RET_OK) { + *bHasRestrictions = 1; /* in this case it is better to check individually */ + DBGPRINTF("Error %d trying to obtain ACL restriction state of '%s'\n", iRet, pszType); + } + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-03-05 */ @@ -1562,12 +1603,14 @@ CODESTARTobjQueryInterface(net) pIf->create_udp_socket = create_udp_socket; pIf->closeUDPListenSockets = closeUDPListenSockets; pIf->isAllowedSender = isAllowedSender; + pIf->isAllowedSender2 = isAllowedSender2; pIf->should_use_so_bsdcompat = should_use_so_bsdcompat; pIf->getLocalHostname = getLocalHostname; pIf->AddPermittedPeer = AddPermittedPeer; pIf->DestructPermittedPeers = DestructPermittedPeers; pIf->PermittedPeerWildcardMatch = PermittedPeerWildcardMatch; pIf->CmpHost = CmpHost; + pIf->HasRestrictions = HasRestrictions; /* data members */ pIf->pACLAddHostnameOnFail = &ACLAddHostnameOnFail; pIf->pACLDontResolve = &ACLDontResolve; diff --git a/runtime/net.h b/runtime/net.h index ec364b1c..101ce79d 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -139,7 +139,7 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ void (*debugListenInfo)(int fd, char *type); int *(*create_udp_socket)(uchar *hostname, uchar *LogPort, int bIsServer); void (*closeUDPListenSockets)(int *finet); - int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); + int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); /* deprecated! */ rsRetVal (*getLocalHostname)(uchar**); int (*should_use_so_bsdcompat)(void); /* permitted peer handling should be replaced by something better (see comments above) */ @@ -148,11 +148,14 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ rsRetVal (*PermittedPeerWildcardMatch)(permittedPeers_t *pPeer, uchar *pszNameToMatch, int *pbIsMatching); /* v5 interface additions */ int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t); + /* v6 interface additions - 2009-11-16 */ + rsRetVal (*HasRestrictions)(uchar *, int *bHasRestrictions); + int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS); /* data members - these should go away over time... TODO */ int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ ENDinterface(net) -#define netCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define netCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* prototypes */ PROTOTYPEObj(net); diff --git a/runtime/netstrms.c b/runtime/netstrms.c index 6b28e7ea..e9ff2568 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -36,6 +36,7 @@ #include "nsd.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "netstrms.h" MODULE_TYPE_LIB @@ -304,6 +305,7 @@ ENDObjClassInit(netstrms) BEGINmodExit CODESTARTmodExit nsselClassExit(); + nspollClassExit(); netstrmsClassExit(); netstrmClassExit(); /* we use this object, so we must exit it after we are finished */ ENDmodExit @@ -322,6 +324,7 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(netstrmClassInit(pModInfo)); CHKiRet(nsselClassInit(pModInfo)); + CHKiRet(nspollClassInit(pModInfo)); CHKiRet(netstrmsClassInit(pModInfo)); ENDmodInit /* vi:set ai: diff --git a/runtime/nsd.h b/runtime/nsd.h index 8668c934..e5b9320b 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -87,4 +87,13 @@ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ ENDinterface(nsdsel) #define nsdselCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface for the epoll call */ +BEGINinterface(nsdpoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nsdpoll_t **ppThis); + rsRetVal (*Destruct)(nsdpoll_t **ppThis); + rsRetVal (*Ctl)(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op); + rsRetVal (*Wait)(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); +ENDinterface(nsdpoll) +#define nsdpollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + #endif /* #ifndef INCLUDED_NSD_H */ diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index fb2e219d..0ee70e56 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -44,6 +44,7 @@ #include "stringbuf.h" #include "errmsg.h" #include "net.h" +#include "datetime.h" #include "nsd_ptcp.h" #include "nsdsel_gtls.h" #include "nsd_gtls.h" @@ -61,6 +62,7 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) +DEFobjCurrIf(datetime) DEFobjCurrIf(nsd_ptcp) static int bGlblSrvrInitDone = 0; /**< 0 - server global init not yet done, 1 - already done */ @@ -129,7 +131,7 @@ readFile(uchar *pszFile, gnutls_datum_t *pBuf) ABORT_FINALIZE(RS_RET_FILE_TOO_LARGE); } - CHKmalloc(pBuf->data = malloc(stat_st.st_size)); + CHKmalloc(pBuf->data = MALLOC(stat_st.st_size)); pBuf->size = stat_st.st_size; if(read(fd, pBuf->data, stat_st.st_size) != stat_st.st_size) { errmsg.LogError(0, RS_RET_IO_ERROR, "error or incomplete read of file '%s'", pszFile); @@ -1016,7 +1018,7 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) } /* get current time for certificate validation */ - if(time(&ttNow) == -1) + if(datetime.GetTime(&ttNow) == -1) ABORT_FINALIZE(RS_RET_SYS_ERR); /* as it looks, we need to validate the expiration dates ourselves... @@ -1482,7 +1484,7 @@ Rcv(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) if(pThis->pszRcvBuf == NULL) { /* we have no buffer, so we need to malloc one */ - CHKmalloc(pThis->pszRcvBuf = malloc(NSD_GTLS_MAX_RCVBUF)); + CHKmalloc(pThis->pszRcvBuf = MALLOC(NSD_GTLS_MAX_RCVBUF)); pThis->lenRcvBuf = -1; } @@ -1697,6 +1699,7 @@ CODESTARTObjClassExit(nsd_gtls) objRelease(nsd_ptcp, LM_NSD_PTCP_FILENAME); objRelease(net, LM_NET_FILENAME); objRelease(glbl, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); ENDObjClassExit(nsd_gtls) @@ -1708,6 +1711,7 @@ ENDObjClassExit(nsd_gtls) BEGINObjClassInit(nsd_gtls, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); CHKiRet(objUse(nsd_ptcp, LM_NSD_PTCP_FILENAME)); diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index 54ee0666..ca00749c 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -48,6 +48,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nsdsel_ptcp.h" +#include "nsdpoll_ptcp.h" #include "nsd_ptcp.h" MODULE_TYPE_LIB @@ -296,12 +297,12 @@ FillRemHost(nsd_ptcp_t *pThis, struct sockaddr *pAddr) * memory consumption) */ len = strlen((char*)szIP) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostIP = malloc(len)) == NULL) + if((pThis->pRemHostIP = MALLOC(len)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pRemHostIP, szIP, len); len = strlen((char*)szHname) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostName = malloc(len)) == NULL) { + if((pThis->pRemHostName = MALLOC(len)) == NULL) { free(pThis->pRemHostIP); /* prevent leak */ pThis->pRemHostIP = NULL; ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -562,6 +563,7 @@ finalize_it: static rsRetVal Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) { + char errStr[1024]; DEFiRet; nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; ISOBJ_TYPE_assert(pThis, nsd_ptcp); @@ -571,7 +573,9 @@ Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf) if(*pLenBuf == 0) { ABORT_FINALIZE(RS_RET_CLOSED); } else if (*pLenBuf < 0) { - ABORT_FINALIZE(RS_RET_ERR); + rs_strerror_r(errno, errStr, sizeof(errStr)); + dbgprintf("error during recv on NSD %p: %s\n", pNsd, errStr); + ABORT_FINALIZE(RS_RET_RCV_ERR); } finalize_it: @@ -821,6 +825,9 @@ ENDObjClassInit(nsd_ptcp) BEGINmodExit CODESTARTmodExit +# ifdef HAVE_EPOLL_CREATE /* module only available if epoll() is supported! */ + nsdpoll_ptcpClassExit(); +# endif nsdsel_ptcpClassExit(); nsd_ptcpClassExit(); ENDmodExit @@ -839,6 +846,9 @@ CODESTARTmodInit /* Initialize all classes that are in our module - this includes ourselfs */ CHKiRet(nsd_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ CHKiRet(nsdsel_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ +# ifdef HAVE_EPOLL_CREATE /* module only available if epoll() is supported! */ + CHKiRet(nsdpoll_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ +# endif ENDmodInit /* vi:set ai: */ diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c new file mode 100644 index 00000000..51006707 --- /dev/null +++ b/runtime/nsdpoll_ptcp.c @@ -0,0 +1,288 @@ +/* nsdpoll_ptcp.c + * + * An implementation of the nsd epoll() interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#ifdef HAVE_EPOLL_CREATE /* this module requires epoll! */ + +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "errmsg.h" +#include "srUtils.h" +#include "nspoll.h" +#include "nsd_ptcp.h" +#include "nsdpoll_ptcp.h" +#include "unlimited_select.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) + + +/* -START------------------------- helpers for event list ------------------------------------ */ + +/* add new entry to list. We assume that the fd is not already present and DO NOT check this! + * Returns newly created entry in pEvtLst. + * Note that we currently need to use level-triggered mode, because the upper layers do not work + * in parallel. As such, in edge-triggered mode we may not get notified, because new data comes + * in after we have read everything that was present. To use ET mode, we need to change the upper + * peers so that they immediately start a new wait before processing the data read. That obviously + * requires more elaborate redesign and we postpone this until the current more simplictic mode has + * been proven OK in practice. + * rgerhards, 2009-11-18 + */ +static inline rsRetVal +addEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, int mode, nsd_ptcp_t *pSock, nsdpoll_epollevt_lst_t **pEvtLst) { + nsdpoll_epollevt_lst_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (nsdpoll_epollevt_lst_t*) malloc(sizeof(nsdpoll_epollevt_lst_t))); + pNew->id = id; + pNew->pUsr = pUsr; + pNew->pSock = pSock; + pNew->event.events = 0; /* TODO: at some time we should be able to use EPOLLET */ + if(mode & NSDPOLL_IN) + pNew->event.events |= EPOLLIN; + if(mode & NSDPOLL_OUT) + pNew->event.events |= EPOLLOUT; + pNew->event.data.u64 = (uint64) pNew; + pNew->pNext = pThis->pRoot; + pThis->pRoot = pNew; + *pEvtLst = pNew; + +finalize_it: + RETiRet; +} + + +/* find and unlink the entry identified by id/pUsr from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +unlinkEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, nsdpoll_epollevt_lst_t **ppEvtLst) { + nsdpoll_epollevt_lst_t *pEvtLst; + nsdpoll_epollevt_lst_t *pPrev = NULL; + DEFiRet; + + pEvtLst = pThis->pRoot; + while(pEvtLst != NULL && !(pEvtLst->id == id && pEvtLst->pUsr == pUsr)) { + pPrev = pEvtLst; + pEvtLst = pEvtLst->pNext; + } + if(pEvtLst == NULL) + ABORT_FINALIZE(RS_RET_NOT_FOUND); + + *ppEvtLst = pEvtLst; + + /* unlink */ + if(pPrev == NULL) + pThis->pRoot = pEvtLst->pNext; + else + pPrev->pNext = pEvtLst->pNext; + +finalize_it: + RETiRet; +} + + +/* destruct the provided element. It must already be unlinked from the list. + * rgerhards, 2009-11-23 + */ +static inline rsRetVal +delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) { + DEFiRet; + free(*ppEvtLst); + *ppEvtLst = NULL; + RETiRet; +} + + +/* -END--------------------------- helpers for event list ------------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */ +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); + pThis->efd = epoll_create1(EPOLL_CLOEXEC); +# else + DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n"); + pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */ +# endif + if(pThis->efd < 0) { + DBGPRINTF("epoll_create1() could not create fd\n"); + ABORT_FINALIZE(RS_RET_IO_ERROR); + } +finalize_it: +ENDobjConstruct(nsdpoll_ptcp) + + +/* destructor for the nsdpoll_ptcp object */ +BEGINobjDestruct(nsdpoll_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nsdpoll_ptcp) +ENDobjDestruct(nsdpoll_ptcp) + + +/* Modify socket set */ +static rsRetVal +Ctl(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd; + nsdpoll_epollevt_lst_t *pEventLst; + int errSave; + char errStr[512]; + DEFiRet; + + if(op == NSDPOLL_ADD) { + dbgprintf("adding nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(addEvent(pThis, id, pUsr, mode, pSock, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_ADD, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + } + } else if(op == NSDPOLL_DEL) { + dbgprintf("removing nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock); + CHKiRet(unlinkEvent(pThis, id, pUsr, &pEventLst)); + if(epoll_ctl(pThis->efd, EPOLL_CTL_DEL, pSock->sock, &pEventLst->event) < 0) { + errSave = errno; + rs_strerror_r(errSave, errStr, sizeof(errStr)); + errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL, + "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n", + pSock->sock, id, pUsr, mode, errStr); + ABORT_FINALIZE(RS_RET_ERR_EPOLL_CTL); + } + CHKiRet(delEvent(&pEventLst)); + } else { + dbgprintf("program error: invalid NSDPOLL_mode %d - ignoring request\n", op); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Wait for io to become ready. After the successful call, idRdy contains the + * id set by the caller for that i/o event, ppUsr is a pointer to a location + * where the user pointer shall be stored. + * TODO: this is a trivial implementation that only polls one event at a time. We + * may later extend it to poll for multiple events, what would cause less + * overhead. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Wait(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr) { + nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll; + nsdpoll_epollevt_lst_t *pOurEvt; + struct epoll_event event; + int nfds; + DEFiRet; + + assert(idRdy != NULL); + assert(ppUsr != NULL); + + nfds = epoll_wait(pThis->efd, &event, 1, timeout); + if(nfds == -1) { + if(errno == EINTR) { + ABORT_FINALIZE(RS_RET_EINTR); + } else { + DBGPRINTF("epoll() returned with error code %d\n", errno); + ABORT_FINALIZE(RS_RET_ERR_EPOLL); + } + } else if(nfds == 0) { + ABORT_FINALIZE(RS_RET_TIMEOUT); + } + + /* we got a valid event, so tell the caller... */ + pOurEvt = (nsdpoll_epollevt_lst_t*) event.data.u64; + *idRdy = pOurEvt->id; + *ppUsr = pOurEvt->pUsr; + +finalize_it: + RETiRet; +} + + +/* ------------------------------ end support for the epoll() interface ------------------------------ */ + + +/* queryInterface function */ +BEGINobjQueryInterface(nsdpoll_ptcp) +CODESTARTobjQueryInterface(nsdpoll_ptcp) + if(pIf->ifVersion != nsdCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpConstruct; + pIf->Destruct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpDestruct; + pIf->Ctl = Ctl; + pIf->Wait = Wait; +finalize_it: +ENDobjQueryInterface(nsdpoll_ptcp) + + +/* exit our class + */ +BEGINObjClassExit(nsdpoll_ptcp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdpoll_ptcp) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(nsdpoll_ptcp) + + +/* Initialize the nsdpoll_ptcp class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nsdpoll_ptcp, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nsdpoll_ptcp) +#endif /* #ifdef HAVE_EPOLL_CREATE this module requires epoll! */ + +/* vi:set ai: + */ diff --git a/runtime/nsdpoll_ptcp.h b/runtime/nsdpoll_ptcp.h new file mode 100644 index 00000000..cea2823d --- /dev/null +++ b/runtime/nsdpoll_ptcp.h @@ -0,0 +1,60 @@ +/* An implementation of the nsd poll interface for plain tcp sockets. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSDPOLL_PTCP_H +#define INCLUDED_NSDPOLL_PTCP_H + +#include "nsd.h" +#if HAVE_SYS_EPOLL_H +# include <sys/epoll.h> +#endif +typedef nsdpoll_if_t nsdpoll_ptcp_if_t; /* we just *implement* this interface */ +/* a helper object to keep track of the epoll event records + * Note that we need to keep track of that list because we need to + * free the events when they are no longer needed. + */ +typedef struct nsdpoll_epollevt_lst_s nsdpoll_epollevt_lst_t; +struct nsdpoll_epollevt_lst_s { +#if HAVE_SYS_EPOLL_H + epoll_event_t event; +#endif + int id; + void *pUsr; + nsd_ptcp_t *pSock; /* our associated netstream driver data */ + nsdpoll_epollevt_lst_t *pNext; +}; + +/* the nsdpoll_ptcp object */ +struct nsdpoll_ptcp_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int efd; /* file descriptor used by epoll */ + nsdpoll_epollevt_lst_t *pRoot; /* Root of the epoll event list */ +}; + +/* interface is defined in nsd.h, we just implement it! */ +#define nsdpoll_ptcpCURR_IF_VERSION nsdCURR_IF_VERSION + +/* prototypes */ +PROTOTYPEObj(nsdpoll_ptcp); + +#endif /* #ifndef INCLUDED_NSDPOLL_PTCP_H */ diff --git a/runtime/nspoll.c b/runtime/nspoll.c new file mode 100644 index 00000000..64927280 --- /dev/null +++ b/runtime/nspoll.c @@ -0,0 +1,195 @@ +/* nspoll.c + * + * This is an io waiter interface utilizing the much-more-efficient poll/epoll API. + * Note that it may not always be available for a given driver. If so, that is reported + * back to the upper peer which then should consult a nssel-based io waiter. + * + * Work on this module begun 2009-11-18 by Rainer Gerhards. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" + +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <string.h> + +#include "rsyslog.h" +#include "obj.h" +#include "module-template.h" +#include "netstrm.h" +#include "nspoll.h" + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) + + +/* load our low-level driver. This must be done before any + * driver-specific functions (allmost all...) can be carried + * out. Note that the driver's .ifIsLoaded is correctly + * initialized by calloc() and we depend on that. Please note that + * we do some name-mangeling. We know that each nsd driver also needs + * a nspoll driver. So we simply append "sel" to the nsd driver name: This, + * of course, means that the driver name must match these rules, but that + * shouldn't be a real problem. + * WARNING: this code is mostly identical to similar code in + * netstrms.c - TODO: abstract it and move it to some common place. + * rgerhards, 2008-04-28 + */ +static rsRetVal +loadDrvr(nspoll_t *pThis) +{ + DEFiRet; + uchar *pBaseDrvrName; + uchar szDrvrName[48]; /* 48 shall be large enough */ + + pBaseDrvrName = pThis->pBaseDrvrName; + if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdpoll_%s", pBaseDrvrName) == sizeof(szDrvrName)) + ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); + CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); + + pThis->Drvr.ifVersion = nsdCURR_IF_VERSION; + /* The pDrvrName+2 below is a hack to obtain the object name. It + * safes us to have yet another variable with the name without "lm" in + * front of it. If we change the module load interface, we may re-think + * about this hack, but for the time being it is efficient and clean + * enough. -- rgerhards, 2008-04-18 + */ + CHKiRet(obj.UseObj(__FILE__, szDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr)); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pDrvrName != NULL) + free(pThis->pDrvrName); + pThis->pDrvrName = NULL; + } + RETiRet; +} + + +/* Standard-Constructor */ +BEGINobjConstruct(nspoll) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(nspoll) + + +/* destructor for the nspoll object */ +BEGINobjDestruct(nspoll) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(nspoll) + if(pThis->pDrvrData != NULL) + pThis->Drvr.Destruct(&pThis->pDrvrData); + + /* and now we must release our driver, if we got one. We use the presence of + * a driver name string as load indicator (because we also need that string + * to release the driver + */ + if(pThis->pDrvrName != NULL) { + obj.ReleaseObj(__FILE__, pThis->pDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr); + free(pThis->pDrvrName); + } +ENDobjDestruct(nspoll) + + +/* ConstructionFinalizer */ +static rsRetVal +ConstructFinalize(nspoll_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + CHKiRet(loadDrvr(pThis)); + CHKiRet(pThis->Drvr.Construct(&pThis->pDrvrData)); +finalize_it: + RETiRet; +} + + +/* Carries out the actual wait (all done in lower layers) + */ +static rsRetVal +Wait(nspoll_t *pThis, int timeout, int *idRdy, void **ppUsr) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + assert(idRdy != NULL); + iRet = pThis->Drvr.Wait(pThis->pDrvrData, timeout, idRdy, ppUsr); + RETiRet; +} + + +/* semantics like the epoll_ctl() function, does the same thing. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Ctl(nspoll_t *pThis, netstrm_t *pStrm, int id, void *pUsr, int mode, int op) { + DEFiRet; + ISOBJ_TYPE_assert(pThis, nspoll); + iRet = pThis->Drvr.Ctl(pThis->pDrvrData, pStrm->pDrvrData, id, pUsr, mode, op); + RETiRet; +} + + +/* queryInterface function */ +BEGINobjQueryInterface(nspoll) +CODESTARTobjQueryInterface(nspoll) + if(pIf->ifVersion != nspollCURR_IF_VERSION) {/* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = nspollConstruct; + pIf->ConstructFinalize = ConstructFinalize; + pIf->Destruct = nspollDestruct; + pIf->Wait = Wait; + pIf->Ctl = Ctl; +finalize_it: +ENDobjQueryInterface(nspoll) + + +/* exit our class + */ +BEGINObjClassExit(nspoll, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nspoll) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(nspoll) + + +/* Initialize the nspoll class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(nspoll, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + DBGPRINTF("doing nspollClassInit\n"); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ +ENDObjClassInit(nspoll) +/* vi:set ai: + */ diff --git a/runtime/nspoll.h b/runtime/nspoll.h new file mode 100644 index 00000000..a77759c0 --- /dev/null +++ b/runtime/nspoll.h @@ -0,0 +1,65 @@ +/* Definitions for the nspoll io activity waiter + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef INCLUDED_NSPOLL_H +#define INCLUDED_NSPOLL_H + +#include "netstrms.h" + +/* some operations to be portable when we do not have epoll() available */ +#define NSDPOLL_ADD 1 +#define NSDPOLL_DEL 2 + +/* and some mode specifiers for waiting on input/output */ +#define NSDPOLL_IN 1 /* EPOLLIN */ +#define NSDPOLL_OUT 2 /* EPOLLOUT */ +/* next is 4, 8, 16, ... - must be bit values, as they are ored! */ + +/* the nspoll object */ +struct nspoll_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + nsd_t *pDrvrData; /**< the driver's data elements */ + uchar *pBaseDrvrName; /**< nsd base driver name to use, or NULL if system default */ + uchar *pDrvrName; /**< full base driver name (set when driver is loaded) */ + nsdpoll_if_t Drvr; /**< our stream driver */ +}; + + +/* interface */ +BEGINinterface(nspoll) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(nspoll_t **ppThis); + rsRetVal (*ConstructFinalize)(nspoll_t *pThis); + rsRetVal (*Destruct)(nspoll_t **ppThis); + rsRetVal (*Wait)(nspoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr); + rsRetVal (*Ctl)(nspoll_t *pNsdpoll, netstrm_t *pStrm, int id, void *pUsr, int mode, int op); + rsRetVal (*IsEPollSupported)(void); /* static method */ +ENDinterface(nspoll) +#define nspollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + +/* prototypes */ +PROTOTYPEObj(nspoll); + +/* the name of our library binary */ +#define LM_NSPOLL_FILENAME LM_NETSTRMS_FILENAME + +#endif /* #ifndef INCLUDED_NSPOLL_H */ diff --git a/runtime/nssel.c b/runtime/nssel.c index d11d5fe1..7c5be3a9 100644 --- a/runtime/nssel.c +++ b/runtime/nssel.c @@ -219,6 +219,7 @@ ENDObjClassExit(nssel) */ BEGINObjClassInit(nssel, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + DBGPRINTF("doing nsselClassInit\n"); CHKiRet(objUse(glbl, CORE_COMPONENT)); /* set our own handlers */ diff --git a/runtime/obj.c b/runtime/obj.c index aebea332..45dac776 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -48,7 +48,7 @@ * * File begun on 2008-01-04 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -90,6 +90,7 @@ #include "cfsysline.h" #include "unicode-helper.h" #include "apc.h" +#include "datetime.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ @@ -1129,7 +1130,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* DEV debug only: dbgprintf("source file %s requests object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ - d_pthread_mutex_lock(&mutObjGlobalOp); + pthread_mutex_lock(&mutObjGlobalOp); if(pIf->ifIsLoaded == 1) { ABORT_FINALIZE(RS_RET_OK); /* we are already set */ @@ -1170,7 +1171,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 1; /* we are happy */ finalize_it: - d_pthread_mutex_unlock(&mutObjGlobalOp); + pthread_mutex_unlock(&mutObjGlobalOp); if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1193,7 +1194,7 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* dev debug only dbgprintf("source file %s releasing object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ - d_pthread_mutex_lock(&mutObjGlobalOp); + pthread_mutex_lock(&mutObjGlobalOp); if(pObjFile == NULL) FINALIZE; /* if it is not a lodable module, we do not need to do anything... */ @@ -1214,7 +1215,7 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 0; /* indicated "no longer valid" */ finalize_it: - d_pthread_mutex_unlock(&mutObjGlobalOp); + pthread_mutex_unlock(&mutObjGlobalOp); if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1330,8 +1331,9 @@ objClassInit(modInfo_t *pModInfo) CHKiRet(objGetObjInterface(&obj)); /* get ourselves ;) */ /* init classes we use (limit to as few as possible!) */ - CHKiRet(apcClassInit(pModInfo)); CHKiRet(errmsgClassInit(pModInfo)); + CHKiRet(datetimeClassInit(pModInfo)); + CHKiRet(apcClassInit(pModInfo)); CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); CHKiRet(moduleClassInit(pModInfo)); diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 8dddc4b8..1d442c61 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -67,31 +67,25 @@ rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries) assert(ppThis != NULL); assert(iNumEntries >= 0); - if((pThis = calloc(1, sizeof(omodStringRequest_t))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis = calloc(1, sizeof(omodStringRequest_t))); /* got the structure, so fill it */ pThis->iNumEntries = iNumEntries; /* allocate string for template name array. The individual strings will be * allocated as the code progresses (we do not yet know the string sizes) */ - if((pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))); + /* allocate the template options array. */ - if((pThis->piTplOpts = calloc(iNumEntries, sizeof(int))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->piTplOpts = calloc(iNumEntries, sizeof(int))); -abort_it: +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis != NULL) { + OMSRdestruct(pThis); + pThis = NULL; + } + } *ppThis = pThis; RETiRet; } @@ -155,7 +149,7 @@ OMSRgetSupportedTplOpts(unsigned long *pOpts) { DEFiRet; assert(pOpts != NULL); - *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY; + *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY | OMSR_TPL_AS_MSG; RETiRet; } diff --git a/runtime/objomsr.h b/runtime/objomsr.h index 75ad0fb8..e59b774f 100644 --- a/runtime/objomsr.h +++ b/runtime/objomsr.h @@ -27,8 +27,12 @@ /* define flags for required template options */ #define OMSR_NO_RQD_TPL_OPTS 0 #define OMSR_RQD_TPL_OPT_SQL 1 +/* only one of OMSR_TPL_AS_ARRAY or _AS_MSG must be specified, if both are given + * results are unpredictable. + */ #define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */ -/* next option is 4, 8, 16, ... */ +#define OMSR_TPL_AS_MSG 4 /* introduced in 5.3.4, 2009-11-02 */ +/* next option is 8, 16, 32, ... */ struct omodStringRequest_s { /* strings requested by output module for doAction() */ int iNumEntries; /* number of array entries for data elements below */ diff --git a/runtime/parser.c b/runtime/parser.c index 810bf42b..40374ae1 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -37,7 +37,13 @@ #include "dirty.h" #include "msg.h" #include "obj.h" +#include "datetime.h" #include "errmsg.h" +#include "parser.h" +#include "ruleset.h" +#include "unicode-helper.h" +#include "dirty.h" +#include "cfsysline.h" /* some defines */ #define DEFUPRI (LOG_USER|LOG_NOTICE) @@ -46,25 +52,166 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) +DEFobjCurrIf(ruleset) /* static data */ +/* config data */ +static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ +static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */ +static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ +static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ + +/* This is the list of all parsers known to us. + * This is also used to unload all modules on shutdown. + */ +parserList_t *pParsLstRoot = NULL; + +/* this is the list of the default parsers, to be used if no others + * are specified. + */ +parserList_t *pDfltParsLst = NULL; + + +/* intialize (but NOT allocate) a parser list. Primarily meant as a hook + * which can be used to extend the list in the future. So far, just sets + * it to NULL. + */ +static rsRetVal +InitParserList(parserList_t **pListRoot) +{ + *pListRoot = NULL; + return RS_RET_OK; +} + + +/* destruct a parser list. The list elements are destroyed, but the parser objects + * themselves are not modified. (That is done at a late stage during rsyslogd + * shutdown and need not be considered here.) + */ +static rsRetVal +DestructParserList(parserList_t **ppListRoot) +{ + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = *ppListRoot; + while(pParsLst != NULL) { + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } + *ppListRoot = NULL; + return RS_RET_OK; +} + -/* this is a dummy class init +/* Add a parser to the list. We use a VERY simple and ineffcient algorithm, + * but it is employed only for a few milliseconds during config processing. So + * I prefer to keep it very simple and with simple data structures. Unfortunately, + * we need to preserve the order, but I don't like to add a tail pointer as that + * would require a container object. So I do the extra work to skip to the tail + * when adding elements... + * rgerhards, 2009-11-03 */ -rsRetVal parserClassInit(void) +static rsRetVal +AddParserToList(parserList_t **ppListRoot, parser_t *pParser) { + parserList_t *pThis; + parserList_t *pTail; DEFiRet; - /* request objects we use */ - CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */ - CHKiRet(objUse(glbl, CORE_COMPONENT)); - CHKiRet(objUse(errmsg, CORE_COMPONENT)); -// TODO: free components! see action.c + CHKmalloc(pThis = MALLOC(sizeof(parserList_t))); + pThis->pParser = pParser; + pThis->pNext = NULL; + + if(*ppListRoot == NULL) { + pThis->pNext = *ppListRoot; + *ppListRoot = pThis; + } else { + /* find tail first */ + for(pTail = *ppListRoot ; pTail->pNext != NULL ; pTail = pTail->pNext) + /* just search, do nothing else */; + /* add at tail */ + pTail->pNext = pThis; + } + +finalize_it: + RETiRet; +} + + +/* find a parser based on the provided name */ +static rsRetVal +FindParser(parser_t **ppParser, uchar *pName) +{ + parserList_t *pThis; + DEFiRet; + + for(pThis = pParsLstRoot ; pThis != NULL ; pThis = pThis->pNext) { + if(ustrcmp(pThis->pParser->pName, pName) == 0) { + *ppParser = pThis->pParser; + FINALIZE; /* found it, iRet still eq. OK! */ + } + } + + iRet = RS_RET_PARSER_NOT_FOUND; + +finalize_it: + RETiRet; +} + + +/* --- END helper functions for parser list handling --- */ + +/* Add a an already existing parser to the default list. As usual, order + * of calls is important (most importantly, that means the legacy parser, + * which can process everything, MUST be added last!). + * rgerhards, 2009-11-04 + */ +static rsRetVal +AddDfltParser(uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + CHKiRet(FindParser(&pParser, pName)); + CHKiRet(AddParserToList(&pDfltParsLst, pParser)); + dbgprintf("Parser '%s' added to default parser set.\n", pName); + +finalize_it: + RETiRet; +} + + + +BEGINobjConstruct(parser) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(parser) + +/* ConstructionFinalizer. The most important chore is to add the parser object + * to our global list of available parsers. + * rgerhards, 2009-11-03 + */ +rsRetVal parserConstructFinalize(parser_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + CHKiRet(AddParserToList(&pParsLstRoot, pThis)); + DBGPRINTF("Parser '%s' added to list of available parsers.\n", pThis->pName); + finalize_it: RETiRet; } +BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(parser) + dbgprintf("destructing parser '%s'\n", pThis->pName); + free(pThis->pName); +ENDobjDestruct(parser) + /* uncompress a received message if it is compressed. * pMsg->pszRawMsg buffer is updated. @@ -96,7 +243,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) */ int ret; iLenDefBuf = glbl.GetMaxLine(); - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1))); + CHKmalloc(deflateBuf = MALLOC(sizeof(uchar) * (iLenDefBuf + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, (int) (lenMsg-1)); @@ -153,7 +300,7 @@ finalize_it: * rgerhards, 2007-09-14 */ static inline rsRetVal -sanitizeMessage(msg_t *pMsg) +SanitizeMsg(msg_t *pMsg) { DEFiRet; uchar *pszMsg; @@ -163,16 +310,12 @@ sanitizeMessage(msg_t *pMsg) size_t iDst; size_t iMaxLine; size_t maxDest; - bool bUpdatedLen = FALSE; + sbool bUpdatedLen = FALSE; uchar szSanBuf[32*1024]; /* buffer used for sanitizing a string */ assert(pMsg != NULL); assert(pMsg->iLenRawMsg > 0); -# ifdef USE_NETZIP - CHKiRet(uncompressMessage(pMsg)); -# endif - pszMsg = pMsg->pszRawMsg; lenMsg = pMsg->iLenRawMsg; @@ -202,6 +345,11 @@ sanitizeMessage(msg_t *pMsg) * needs sanitation than to do the sanitation in any case. So we first do * this and terminate when it is not needed - which is expectedly the case * for the vast majority of messages. -- rgerhards, 2009-06-15 + * Note that we do NOT check here if tab characters are to be escaped or + * not. I expect this functionality to be seldomly used and thus I do not + * like to pay the performance penalty. So the penalty is only with those + * that actually use it, because we may call the sanitizer without actual + * need below (but it then still will work perfectly well!). -- rgerhards, 2009-11-27 */ int bNeedSanitize = 0; for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { @@ -210,6 +358,9 @@ sanitizeMessage(msg_t *pMsg) bNeedSanitize = 1; break; } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + bNeedSanitize = 1; + break; } } @@ -227,23 +378,31 @@ sanitizeMessage(msg_t *pMsg) if(maxDest < sizeof(szSanBuf)) pDst = szSanBuf; else - CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ - if(iscntrl((int) pszMsg[iSrc])) { + if(iscntrl((int) pszMsg[iSrc]) && (pszMsg[iSrc] != '\t' || bEscapeTab)) { /* note: \0 must always be escaped, the rest of the code currently * can not handle it! -- rgerhards, 2009-08-26 */ if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { - /* we are configured to escape control characters. Please note - * that this most probably break non-western character sets like - * Japanese, Korean or Chinese. rgerhards, 2007-07-17 + /* we are configured to escape control characters. Please note + * that this most probably break non-western character sets like + * Japanese, Korean or Chinese. rgerhards, 2007-07-17 + */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); + } + } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { + /* In this case, we also do the conversion. Note that this most + * probably breaks European languages. -- rgerhards, 2010-01-27 */ pDst[iDst++] = cCCEscapeChar; pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); - } } else { pDst[iDst++] = pszMsg[iSrc]; } @@ -260,27 +419,18 @@ finalize_it: RETiRet; } - -/* Parse a received message. The object's rawmsg property is taken and - * parsed according to the relevant standards. This can later be - * extended to support configured parsers. - * rgerhards, 2008-10-09 +/* A standard parser to parse out the PRI. This is made available in + * this module as it is expected that allmost all parsers will need + * that functionality and so they do not need to implement it themsleves. */ -rsRetVal parseMsg(msg_t *pMsg) +static inline rsRetVal +ParsePRI(msg_t *pMsg) { - DEFiRet; - uchar *msg; int pri; + uchar *msg; int lenMsg; int iPriText; - - if(pMsg->iLenRawMsg == 0) - ABORT_FINALIZE(RS_RET_EMPTY_MSG); - - CHKiRet(sanitizeMessage(pMsg)); - - /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ - DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, getRcvFrom(pMsg), pMsg->pszRawMsg); + DEFiRet; /* pull PRI */ lenMsg = pMsg->iLenRawMsg; @@ -309,31 +459,252 @@ rsRetVal parseMsg(msg_t *pMsg) pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); + RETiRet; +} + + +/* Parse a received message. The object's rawmsg property is taken and + * parsed according to the relevant standards. This can later be + * extended to support configured parsers. + * rgerhards, 2008-10-09 + */ +static rsRetVal +ParseMsg(msg_t *pMsg) +{ + rsRetVal localRet = RS_RET_ERR; + parserList_t *pParserList; + parser_t *pParser; + sbool bIsSanitized; + sbool bPRIisParsed; + static int iErrMsgRateLimiter = 0; + DEFiRet; + + if(pMsg->iLenRawMsg == 0) + ABORT_FINALIZE(RS_RET_EMPTY_MSG); - /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have - * a traditional syslog message or one formatted according to syslog-protocol. - * We need to apply different parsers depending on that. We use the - * -protocol VERSION field for the detection. +# ifdef USE_NETZIP + CHKiRet(uncompressMessage(pMsg)); +# endif + + /* we take the risk to print a non-sanitized string, because this is the best we can get + * (and that functionality is too important for debugging to drop it...). + */ + DBGPRINTF("msg parser: flags %x, from '%s', msg '%.60s'\n", pMsg->msgFlags, + (pMsg->msgFlags & NEEDS_DNSRESOL) ? UCHAR_CONSTANT("~NOTRESOLVED~") : getRcvFrom(pMsg), + pMsg->pszRawMsg); + + /* we now need to go through our list of parsers and see which one is capable of + * parsing the message. Note that the first parser that requires message sanitization + * will cause it to happen. After that, access to the unsanitized message is no + * loger possible. */ - if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); - setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + pParserList = ruleset.GetParserList(pMsg); + if(pParserList == NULL) { + pParserList = pDfltParsLst; + } + DBGPRINTF("parse using parser list %p%s.\n", pParserList, + (pParserList == pDfltParsLst) ? " (the default list)" : ""); + + bIsSanitized = FALSE; + bPRIisParsed = FALSE; + while(pParserList != NULL) { + pParser = pParserList->pParser; + if(pParser->bDoSanitazion && bIsSanitized == FALSE) { + CHKiRet(SanitizeMsg(pMsg)); + if(pParser->bDoPRIParsing && bPRIisParsed == FALSE) { + CHKiRet(ParsePRI(pMsg)); + bPRIisParsed = TRUE; + } + bIsSanitized = TRUE; } - } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); - setProtocolVersion(pMsg, 0); - if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + localRet = pParser->pModule->mod.pm.parse(pMsg); + dbgprintf("Parser '%s' returned %d\n", pParser->pName, localRet); + if(localRet != RS_RET_COULD_NOT_PARSE) + break; + pParserList = pParserList->pNext; + } + + /* We need to log a warning message and drop the message if we did not find a parser. + * Note that we log at most the first 1000 message, as this may very well be a problem + * that causes a message generation loop. We do not synchronize that counter, it doesn't + * matter if we log a handful messages more than we should... + */ + if(localRet != RS_RET_OK) { + if(++iErrMsgRateLimiter > 1000) { + errmsg.LogError(0, localRet, "Error: one message could not be processed by " + "any parser, message is being discarded (start of raw msg: '%.50s')", + pMsg->pszRawMsg); } + DBGPRINTF("No parser could process the message (state %d), we need to discard it.\n", localRet); + ABORT_FINALIZE(localRet); } - /* finalize message object */ + /* "finalize" message object */ pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */ finalize_it: RETiRet; } + +/* set the parser name - string is copied over, call can continue to use it, + * but must free it if desired. + */ +static rsRetVal +SetName(parser_t *pThis, uchar *name) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + assert(name != NULL); + + if(pThis->pName != NULL) { + free(pThis->pName); + pThis->pName = NULL; + } + + CHKmalloc(pThis->pName = ustrdup(name)); + +finalize_it: + RETiRet; +} + + +/* set a pointer to "our" module. Note that no module + * pointer must already be set. + */ +static rsRetVal +SetModPtr(parser_t *pThis, modInfo_t *pMod) +{ + ISOBJ_TYPE_assert(pThis, parser); + assert(pMod != NULL); + assert(pThis->pModule == NULL); + pThis->pModule = pMod; + return RS_RET_OK; +} + + +/* Specify if we should do standard message sanitazion before we pass the data + * down to the parser. + */ +static rsRetVal +SetDoSanitazion(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoSanitazion = bDoIt; + return RS_RET_OK; +} + + +/* Specify if we should do standard PRI parsing before we pass the data + * down to the parser module. + */ +static rsRetVal +SetDoPRIParsing(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoPRIParsing = bDoIt; + return RS_RET_OK; +} + + +/* queryInterface function-- rgerhards, 2009-11-03 + */ +BEGINobjQueryInterface(parser) +CODESTARTobjQueryInterface(parser) + if(pIf->ifVersion != parserCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = parserConstruct; + pIf->ConstructFinalize = parserConstructFinalize; + pIf->Destruct = parserDestruct; + pIf->SetName = SetName; + pIf->SetModPtr = SetModPtr; + pIf->SetDoSanitazion = SetDoSanitazion; + pIf->SetDoPRIParsing = SetDoPRIParsing; + pIf->ParseMsg = ParseMsg; + pIf->SanitizeMsg = SanitizeMsg; + pIf->InitParserList = InitParserList; + pIf->DestructParserList = DestructParserList; + pIf->AddParserToList = AddParserToList; + pIf->AddDfltParser = AddDfltParser; + pIf->FindParser = FindParser; +finalize_it: +ENDobjQueryInterface(parser) + + + +/* Reset config variables to default values. + * rgerhards, 2007-07-17 + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + cCCEscapeChar = '#'; + bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bEscape8BitChars = 0; /* default is to escape control characters */ + bEscapeTab = 1; /* default is to escape control characters */ + bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ + + return RS_RET_OK; +} + +/* This destroys the master parserlist and all of its parser entries. MUST only be + * done when the module is shut down. Parser modules are NOT unloaded, rsyslog + * does that at a later stage for all dynamically loaded modules. + */ +static void +destroyMasterParserList(void) +{ + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = pParsLstRoot; + while(pParsLst != NULL) { + parserDestruct(&pParsLst->pParser); + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } +} + +/* Exit our class. + * rgerhards, 2009-11-04 + */ +BEGINObjClassExit(parser, OBJ_IS_CORE_MODULE) /* class, version */ + DestructParserList(&pDfltParsLst); + destroyMasterParserList(); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDObjClassExit(parser) + + +/* Initialize the parser class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2009-11-02 + */ +BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + + CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); + + InitParserList(&pParsLstRoot); + InitParserList(&pDfltParsLst); +ENDObjClassInit(parser) + diff --git a/runtime/parser.h b/runtime/parser.h index cec9c083..bdd572cb 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -1,8 +1,6 @@ /* header for parser.c - * This is not yet an object, but contains all those code necessary to - * parse syslog messages. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -21,10 +19,53 @@ * * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. */ -#ifndef INCLUDED_PARSE_H -#define INCLUDED_PARSE_H +#ifndef INCLUDED_PARSER_H +#define INCLUDED_PARSER_H -extern rsRetVal parserClassInit(void); -extern rsRetVal parseMsg(msg_t*); -#endif /* #ifndef INCLUDED_PARSE_H */ +/* we create a small helper object, a list of parsers, that we can use to + * build a chain of them whereever this is needed (initially thought to be + * used in ruleset.c as well as ourselvs). + */ +struct parserList_s { + parser_t *pParser; + parserList_t *pNext; +}; + + +/* the parser object, a dummy because we have only static methods */ +struct parser_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *pName; /* name of this parser */ + modInfo_t *pModule; /* pointer to parser's module */ + sbool bDoSanitazion; /* do standard message sanitazion before calling parser? */ + sbool bDoPRIParsing; /* do standard PRI parsing before calling parser? */ +}; + +/* interfaces */ +BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(var); + rsRetVal (*Construct)(parser_t **ppThis); + rsRetVal (*ConstructFinalize)(parser_t *pThis); + rsRetVal (*Destruct)(parser_t **ppThis); + rsRetVal (*SetName)(parser_t *pThis, uchar *name); + rsRetVal (*SetModPtr)(parser_t *pThis, modInfo_t *pMod); + rsRetVal (*SetDoSanitazion)(parser_t *pThis, int); + rsRetVal (*SetDoPRIParsing)(parser_t *pThis, int); + rsRetVal (*FindParser)(parser_t **ppThis, uchar*name); + rsRetVal (*InitParserList)(parserList_t **pListRoot); + rsRetVal (*DestructParserList)(parserList_t **pListRoot); + rsRetVal (*AddParserToList)(parserList_t **pListRoot, parser_t *pParser); + /* static functions */ + rsRetVal (*ParseMsg)(msg_t *pMsg); + rsRetVal (*SanitizeMsg)(msg_t *pMsg); + rsRetVal (*AddDfltParser)(uchar *); +ENDinterface(parser) +#define parserCURR_IF_VERSION 1 /* increment whenever you change the interface above! */ + + +/* prototypes */ +PROTOTYPEObj(parser); + + +#endif /* #ifndef INCLUDED_PARSER_H */ diff --git a/runtime/prop.c b/runtime/prop.c index 7f2a56ff..d925bb43 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -85,7 +85,7 @@ static rsRetVal SetString(prop_t *pThis, uchar *psz, int len) if(len < CONF_PROP_BUFSIZE) { memcpy(pThis->szVal.sz, psz, len + 1); } else { - CHKmalloc(pThis->szVal.psz = malloc(len + 1)); + CHKmalloc(pThis->szVal.psz = MALLOC(len + 1)); memcpy(pThis->szVal.psz, psz, len + 1); } diff --git a/runtime/queue.c b/runtime/queue.c index bedefb77..60d17086 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -8,7 +8,11 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * NOTE: as of 2009-04-22, I have begin to remove the qqueue* prefix from static + * function names - this makes it really hard to read and does not provide much + * benefit, at least I (now) think so... + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -51,71 +55,175 @@ #include "wti.h" #include "msg.h" #include "atomic.h" +#include "errmsg.h" +#include "datetime.h" +#include "unicode-helper.h" +#include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS # include <sched.h> -# define pthread_yield() sched_yield() #endif /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(strm) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) /* forward-definitions */ -static rsRetVal qqueueChkPersist(qqueue_t *pThis); -static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); -static rsRetVal qqueueRateLimiter(qqueue_t *pThis); +static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates); +static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); -static int qqueueIsIdleDA(qqueue_t *pThis); -static rsRetVal qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); -static rsRetVal qqueueConsumerCancelCleanup(void *arg1, void *arg2); -static rsRetVal qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); +static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); +static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti); +static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti); +static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub); +static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 #define QUEUE_NO_CHECKPOINT 0 +/*********************************************************************** + * we need a private data structure, the "to-delete" list. As C does + * not provide any partly private data structures, we implement this + * structure right here inside the module. + * Note that this list must always be kept sorted based on a unique + * dequeue ID (which is monotonically increasing). + * rgerhards, 2009-05-18 + ***********************************************************************/ + +/* generate next uniqueue dequeue ID. Note that uniqueness is only required + * on a per-queue basis and while this instance runs. So a stricly monotonically + * increasing counter is sufficient (if enough bits are used). + */ +static inline qDeqID getNextDeqID(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->deqIDAdd++; +} + + +/* return the top element of the to-delete list or NULL, if the + * list is empty. + */ +static inline toDeleteLst_t *tdlPeek(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->toDeleteLst; +} + + +/* remove the top element of the to-delete list. Nothing but the + * element itself is destroyed. Must not be called when the list + * is empty. + */ +static inline rsRetVal tdlPop(qqueue_t *pQueue) +{ + toDeleteLst_t *pRemove; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + pRemove = pQueue->toDeleteLst; + pQueue->toDeleteLst = pQueue->toDeleteLst->pNext; + free(pRemove); + + RETiRet; +} + + +/* Add a new to-delete list entry. The function allocates the data + * structure, populates it with the values provided and links the new + * element into the correct place inside the list. + */ +static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElemDeq) +{ + toDeleteLst_t *pNew; + toDeleteLst_t *pPrev; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + CHKmalloc(pNew = MALLOC(sizeof(toDeleteLst_t))); + pNew->deqID = deqID; + pNew->nElemDeq = nElemDeq; + + /* now find right spot */ + for( pPrev = pQueue->toDeleteLst + ; pPrev != NULL && deqID > pPrev->deqID + ; pPrev = pPrev->pNext) { + /*JUST SEARCH*/; + } + + if(pPrev == NULL) { + pNew->pNext = pQueue->toDeleteLst; + pQueue->toDeleteLst = pNew; + } else { + pNew->pNext = pPrev->pNext; + pPrev->pNext = pNew; + } + +finalize_it: + RETiRet; +} + + /* methods */ -/* get the overall queue size, which includes ungotten objects. Must only be called +/* get the physical queue size. Must only be called * while mutex is locked! * rgerhards, 2008-01-29 */ static inline int -qqueueGetOverallQueueSize(qqueue_t *pThis) +getPhysicalQueueSize(qqueue_t *pThis) { -#if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */ -BEGINfunc -dbgoprint((obj_t*) pThis, "queue size: %d (regular %d, ungotten %d)\n", - pThis->iQueueSize + pThis->iUngottenObjs, pThis->iQueueSize, pThis->iUngottenObjs); -ENDfunc -#endif - return pThis->iQueueSize + pThis->iUngottenObjs; + return pThis->iQueueSize; } +/* get the logical queue size (that is store size minus logically dequeued elements). + * Must only be called while mutex is locked! + * rgerhards, 2009-05-19 + */ +static inline int +getLogicalQueueSize(qqueue_t *pThis) +{ + return pThis->iQueueSize - pThis->nLogDeq; +} + + + /* This function drains the queue in cases where this needs to be done. The most probable * reason is a HUP which needs to discard data (because the queue is configured to be lossy). * During a shutdown, this is typically not needed, as the OS frees up ressources and does * this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21 * This function returns void, as it makes no sense to communicate an error back, even if * it happens. + * This functions works "around" the regular deque mechanism, because it is only used to + * clean up (in cases where message loss is acceptable). */ static inline void queueDrain(qqueue_t *pThis) { void *pUsr; - ASSERT(pThis != NULL); + BEGINfunc + DBGOPRINT((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ while(ATOMIC_DEC_AND_FETCH(&pThis->iQueueSize, &pThis->mutQueueSize) > 0) { - pThis->qDel(pThis, &pUsr); + pThis->qDeq(pThis, &pUsr); if(pUsr != NULL) { objDestruct(pUsr); } + pThis->qDel(pThis); } + ENDfunc } @@ -126,7 +234,8 @@ static inline void queueDrain(qqueue_t *pThis) * this point in time. The mutex must be locked when * ths function is called. -- rgerhards, 2008-01-25 */ -static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) +static inline rsRetVal +qqueueAdviseMaxWorkers(qqueue_t *pThis) { DEFiRet; int iMaxWorkers; @@ -134,96 +243,17 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); if(!pThis->bEnqOnly) { - if(pThis->bRunsDA) { - /* if we have not yet reached the high water mark, there is no need to start a - * worker. -- rgerhards, 2008-01-26 - */ - if(qqueueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ - } + if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) { + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } else { - if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { + if(getLogicalQueueSize(pThis) == 0) { + iMaxWorkers = 0; + } else if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { iMaxWorkers = 1; } else { - iMaxWorkers = qqueueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } - wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ - } - } - - RETiRet; -} - - -/* wait until we have a fully initialized DA queue. Sometimes, we need to - * sync with it, as we expect it for some function. - * rgerhards, 2008-02-27 - */ -static rsRetVal -qqueueWaitDAModeInitialized(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - - while(pThis->bRunsDA != 2) { - d_pthread_cond_wait(&pThis->condDAReady, pThis->mut); - } - - RETiRet; -} - - -/* Destruct DA queue. This is the last part of DA-to-normal-mode - * transistion. This is called asynchronously and some time quite a - * while after the actual transistion. The key point is that we need to - * do it at some later time, because we need to destruct the DA queue. That, - * however, can not be done in a thread that has been signalled - * This is to be called when we revert back to our own queue. - * This function must be called with the queue mutex locked (the wti - * class ensures this). - * rgerhards, 2008-01-15 - */ -static rsRetVal -qqueueTurnOffDAMode(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - - /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need - * to wait for its startup... -- rgerhards, 2008-01-25 - */ - qqueueWaitDAModeInitialized(pThis); - - /* if we need to pull any data that we still need from the (child) disk queue, - * now would be the time to do so. At present, we do not need this, but I'd like to - * keep that comment if future need arises. - */ - - /* we need to check if the DA queue is empty because the DA worker may simply have - * terminated do to no new messages arriving. That does not, however, mean that the - * DA queue is empty. If there is still data in that queue, we do nothing and leave - * that for a later incarnation of this function (it will be called multiple times - * during the lifetime of DA-mode, depending on how often the DA worker receives an - * inactivity timeout. -- rgerhards, 2008-01-25 - */ - if(pThis->pqDA->iQueueSize == 0) { - pThis->bRunsDA = 0; /* tell the world we are back in non-DA mode */ - /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, - * this will be quick. - */ - qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ - dbgoprint((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n", - iRet); - /* now we need to check if the regular queue has some messages. This may be the case - * when it is waiting that the high water mark is reached again. If so, we need to start up - * a regular worker. -- rgerhards, 2008-01-26 - */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - qqueueAdviseMaxWorkers(pThis); + wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); } } @@ -246,37 +276,26 @@ qqueueChkIsDA(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pszFilePrefix != NULL) { pThis->bIsDA = 1; - dbgoprint((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); + DBGOPRINT((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); } else { - dbgoprint((obj_t*) pThis, "is NOT disk-assisted\n"); + DBGOPRINT((obj_t*) pThis, "is NOT disk-assisted\n"); } RETiRet; } -/* Start disk-assisted queue mode. All internal settings are changed. This is supposed - * to be called from the DA worker, which must have been started before. The most important - * chore of this function is to create the DA queue object. If that function fails, - * the DA worker should return with an appropriate state, which in turn should lead to - * a re-set to non-DA mode in the Enq process. The queue mutex must be locked when this - * function is called, else a number of races will happen. - * Please note that this function may be called *while* we in DA mode. This is due to the - * fact that the DA worker calls it and the DA worker may be suspended (and restarted) due - * to inactivity timeouts. +/* Start disk-assisted queue mode. * rgerhards, 2008-01-15 */ static rsRetVal -qqueueStartDA(qqueue_t *pThis) +StartDA(qqueue_t *pThis) { DEFiRet; uchar pszDAQName[128]; ISOBJ_TYPE_assert(pThis, qqueue); - if(pThis->bRunsDA == 2) /* check if already in (fully initialized) DA mode... */ - FINALIZE; /* ... then we are already done! */ - /* create message queue */ CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); @@ -298,39 +317,22 @@ qqueueStartDA(qqueue_t *pThis) CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); - CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, pThis->toQShutdown)); CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0)); - if(pThis->toQShutdown == 0) { - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ - } else { - /* we use the shortest possible shutdown (0 is endless!) because when we run on disk AND - * have an obviously large backlog, we can't finish it in any case. So there is no point - * in holding shutdown longer than necessary. -- rgerhards, 2008-01-15 - */ - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 1)); - } iRet = qqueueStart(pThis->pqDA); /* file not found is expected, that means it is no previous QIF available */ - if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) + if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) { + errno = 0; /* else an errno is shown in errmsg! */ + errmsg.LogError(errno, iRet, "error starting up disk queue, using pure in-memory mode"); + pThis->bIsDA = 0; /* disable memory mode */ FINALIZE; /* something is wrong */ + } - /* as we are right now starting DA mode because we are so busy, it is - * extremely unlikely that any regular worker is sleeping on empty queue. HOWEVER, - * we want to be on the safe side, and so we awake anyone that is waiting - * on one. So even if the scheduler plays badly with us, things should be - * quite well. -- rgerhards, 2008-01-15 - */ - wtpWakeupWrkr(pThis->pWtpReg); /* awake all workers, but not ourselves ;) */ - - pThis->bRunsDA = 2; /* we are now in DA mode, but not fully initialized */ - pThis->bChildIsDone = 0;/* set to 1 when child's worker detect queue is finished */ - pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */ - - dbgoprint((obj_t*) pThis, "is now running in disk assisted mode, disk queue 0x%lx\n", + DBGOPRINT((obj_t*) pThis, "DA queue initialized, disk queue 0x%lx\n", qqueueGetID(pThis->pqDA)); finalize_it: @@ -338,7 +340,7 @@ finalize_it: if(pThis->pqDA != NULL) { qqueueDestruct(&pThis->pqDA); } - dbgoprint((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); + DBGOPRINT((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); pThis->bIsDA = 0; } @@ -352,8 +354,8 @@ finalize_it: * If this function fails (should not happen), DA mode is not turned on. * rgerhards, 2008-01-16 */ -static inline rsRetVal -qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +static rsRetVal +InitDA(qqueue_t *pThis, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -366,82 +368,30 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) * is intentional. We assume that when we need it once, we may also need it on another * occasion. Ressources used are quite minimal when no worker is running. * rgerhards, 2008-01-24 + * NOTE: this is the DA worker *pool*, not the DA queue! */ - if(pThis->pWtpDA == NULL) { - lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DA", obj.GetName((obj_t*) pThis)); - CHKiRet(wtpConstruct (&pThis->pWtpDA)); - CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerDA)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) qqueueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); - CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); - CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); - CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); - CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); - CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); - CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); - } + lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis)); + CHKiRet(wtpConstruct (&pThis->pWtpDA)); + CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerDA)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); + CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); + CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); + CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); + CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); + CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); + CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); /* if we reach this point, we have a "good" DA worker pool */ - /* indicate we now run in DA mode - this is reset by the DA worker if it fails */ - pThis->bRunsDA = 1; - pThis->bDAEnqOnly = bEnqOnly; - - /* now we must now adivse the wtp that we need one worker. If none is yet active, - * that will also start one up. If we forgot that step, everything would be stalled - * until the next enqueue request. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues alsways have just one worker max */ - -finalize_it: - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - RETiRet; -} - - -/* check if we need to start disk assisted mode and send some signals to - * keep it running if we are already in it. It also checks if DA mode is - * partially initialized, in which case it waits for initialization to - * complete. - * rgerhards, 2008-01-14 - */ -static inline rsRetVal -qqueueChkStrtDA(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - /* if we do not hit the high water mark, we have nothing to do */ - if(qqueueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) - ABORT_FINALIZE(RS_RET_OK); - - if(pThis->bRunsDA) { - /* then we need to signal that we are at the high water mark again. If that happens - * on our way down the queue, that doesn't matter, because then nobody is waiting - * on the condition variable. - * (Remember that a DA queue stops draining the queue once it has reached the low - * water mark and restarts it when the high water mark is reached again - this is - * what this code here is responsible for. Please note that all workers may have been - * terminated due to the inactivity timeout, thus we need to advise the pool that - * we need at least one). - */ - dbgoprint((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n", - qqueueGetOverallQueueSize(pThis)); - qqueueAdviseMaxWorkers(pThis); - } else { - /* this is the case when we are currently not running in DA mode. So it is time - * to turn it back on. - */ - dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", - qqueueGetOverallQueueSize(pThis)); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ + /* now construct the actual queue (if it does not already exist) */ + if(pThis->pqDA == NULL) { + CHKiRet(StartDA(pThis)); } finalize_it: + END_MTX_PROTECTED_OPERATIONS(pThis->mut); RETiRet; } @@ -465,10 +415,11 @@ static rsRetVal qConstructFixedArray(qqueue_t *pThis) if(pThis->iMaxQueueSize == 0) ABORT_FINALIZE(RS_RET_QSIZE_ZERO); - if((pThis->tVars.farray.pBuf = malloc(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { + if((pThis->tVars.farray.pBuf = MALLOC(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } + pThis->tVars.farray.deqhead = 0; pThis->tVars.farray.head = 0; pThis->tVars.farray.tail = 0; @@ -486,9 +437,7 @@ static rsRetVal qDestructFixedArray(qqueue_t *pThis) ASSERT(pThis != NULL); queueDrain(pThis); /* discard any remaining queue entries */ - - if(pThis->tVars.farray.pBuf != NULL) - free(pThis->tVars.farray.pBuf); + free(pThis->tVars.farray.pBuf); RETiRet; } @@ -507,76 +456,37 @@ static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) RETiRet; } -static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out) + +static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out) { DEFiRet; ASSERT(pThis != NULL); - *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.head]; + *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.deqhead]; - pThis->tVars.farray.head++; - if (pThis->tVars.farray.head == pThis->iMaxQueueSize) - pThis->tVars.farray.head = 0; + pThis->tVars.farray.deqhead++; + if (pThis->tVars.farray.deqhead == pThis->iMaxQueueSize) + pThis->tVars.farray.deqhead = 0; RETiRet; } -/* -------------------- linked list -------------------- */ - -/* first some generic functions which are also used for the unget linked list */ - -static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) +static rsRetVal qDelFixedArray(qqueue_t *pThis) { DEFiRet; - qLinkedList_t *pEntry; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - pEntry->pNext = NULL; - pEntry->pUsr = pUsr; + ASSERT(pThis != NULL); - if(*ppRoot == NULL) { - *ppRoot = *ppLast = pEntry; - } else { - (*ppLast)->pNext = pEntry; - *ppLast = pEntry; - } + pThis->tVars.farray.head++; + if (pThis->tVars.farray.head == pThis->iMaxQueueSize) + pThis->tVars.farray.head = 0; -finalize_it: RETiRet; } -static inline rsRetVal qqueueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) -{ - DEFiRet; - qLinkedList_t *pEntry; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - ASSERT(ppUsr != NULL); - ASSERT(*ppRoot != NULL); - - pEntry = *ppRoot; - *ppUsr = pEntry->pUsr; - - if(*ppRoot == *ppLast) { - *ppRoot = NULL; - *ppLast = NULL; - } else { - *ppRoot = pEntry->pNext; - } - free(pEntry); - - RETiRet; -} -/* end generic functions which are also used for the unget linked list */ +/* -------------------- linked list -------------------- */ static rsRetVal qConstructLinkedList(qqueue_t *pThis) @@ -585,8 +495,9 @@ static rsRetVal qConstructLinkedList(qqueue_t *pThis) ASSERT(pThis != NULL); - pThis->tVars.linklist.pRoot = 0; - pThis->tVars.linklist.pLast = 0; + pThis->tVars.linklist.pDeqRoot = NULL; + pThis->tVars.linklist.pDelRoot = NULL; + pThis->tVars.linklist.pLast = NULL; qqueueChkIsDA(pThis); @@ -609,54 +520,59 @@ static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) { - DEFiRet; - - iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); -#if 0 qLinkedList_t *pEntry; + DEFiRet; - ASSERT(pThis != NULL); - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc((pEntry = (qLinkedList_t*) MALLOC(sizeof(qLinkedList_t)))); pEntry->pNext = NULL; pEntry->pUsr = pUsr; - if(pThis->tVars.linklist.pRoot == NULL) { - pThis->tVars.linklist.pRoot = pThis->tVars.linklist.pLast = pEntry; + if(pThis->tVars.linklist.pDelRoot == NULL) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = pEntry; } else { pThis->tVars.linklist.pLast->pNext = pEntry; pThis->tVars.linklist.pLast = pEntry; } + if(pThis->tVars.linklist.pDeqRoot == NULL) { + pThis->tVars.linklist.pDeqRoot = pEntry; + } + finalize_it: -#endif RETiRet; } -static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr) + +static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr) { - DEFiRet; - iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); -#if 0 qLinkedList_t *pEntry; + DEFiRet; - ASSERT(pThis != NULL); - ASSERT(pThis->tVars.linklist.pRoot != NULL); - - pEntry = pThis->tVars.linklist.pRoot; + pEntry = pThis->tVars.linklist.pDeqRoot; + ISOBJ_TYPE_assert(pEntry->pUsr, msg); *ppUsr = pEntry->pUsr; + pThis->tVars.linklist.pDeqRoot = pEntry->pNext; + + RETiRet; +} + - if(pThis->tVars.linklist.pRoot == pThis->tVars.linklist.pLast) { - pThis->tVars.linklist.pRoot = NULL; - pThis->tVars.linklist.pLast = NULL; +static rsRetVal qDelLinkedList(qqueue_t *pThis) +{ + qLinkedList_t *pEntry; + DEFiRet; + + pEntry = pThis->tVars.linklist.pDelRoot; + + if(pThis->tVars.linklist.pDelRoot == pThis->tVars.linklist.pLast) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = NULL; } else { - pThis->tVars.linklist.pRoot = pEntry->pNext; + pThis->tVars.linklist.pDelRoot = pEntry->pNext; } + free(pEntry); -#endif RETiRet; } @@ -676,44 +592,6 @@ finalize_it: } -/* This method checks if we have a QIF file for the current queue (no matter of - * queue mode). Returns RS_RET_OK if we have a QIF file or an error status otherwise. - * rgerhards, 2008-01-15 - */ -static rsRetVal -qqueueHaveQIF(qqueue_t *pThis) -{ - DEFiRet; - uchar pszQIFNam[MAXFNAME]; - size_t lenQIFNam; - struct stat stat_buf; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pszFilePrefix == NULL) - ABORT_FINALIZE(RS_RET_NO_FILEPREFIX); - - /* Construct file name */ - lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", - (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - - /* check if the file exists */ - if(stat((char*) pszQIFNam, &stat_buf) == -1) { - if(errno == ENOENT) { - dbgoprint((obj_t*) pThis, "no .qi file found\n"); - ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); - } else { - dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno); - ABORT_FINALIZE(RS_RET_IO_ERROR); - } - } - /* If we reach this point, we have a .qi file */ - -finalize_it: - RETiRet; -} - - /* The method loads the persistent queue information. * rgerhards, 2008-01-11 */ @@ -725,8 +603,6 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; struct stat stat_buf; - int iUngottenObjs; - obj_t *pUsr; ISOBJ_TYPE_assert(pThis, qqueue); @@ -737,10 +613,10 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* check if the file exists */ if(stat((char*) pszQIFNam, &stat_buf) == -1) { if(errno == ENOENT) { - dbgoprint((obj_t*) pThis, "clean startup, no .qi file found\n"); + DBGOPRINT((obj_t*) pThis, "clean startup, no .qi file found\n"); ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); } else { - dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno); + DBGOPRINT((obj_t*) pThis, "error %d trying to access .qi file\n", errno); ABORT_FINALIZE(RS_RET_IO_ERROR); } } @@ -756,25 +632,22 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* first, we try to read the property bag for ourselfs */ CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF)); - /* then the ungotten object queue */ - iUngottenObjs = pThis->iUngottenObjs; - pThis->iUngottenObjs = 0; /* will be incremented when we add objects! */ - - while(iUngottenObjs > 0) { - /* fill the queue from disk */ - CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL)); - qqueueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); - --iUngottenObjs; /* one less */ - } - - /* and now the stream objects (some order as when persisted!) */ + /* then the stream objects (same order as when persisted!) */ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, + CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); + /* create a duplicate for the read "pointer". + */ + + CHKiRet(strm.Dup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */ + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite)); - CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pRead)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDel)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDeq)); /* OK, we could successfully read the file, so we now can request that it be * deleted when we are done with the persisted information. @@ -786,7 +659,7 @@ finalize_it: strm.Destruct(&psQIF); if(iRet != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", + DBGOPRINT((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", iRet); } @@ -826,18 +699,26 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); - CHKiRet(strm.Construct(&pThis->tVars.disk.pRead)); - CHKiRet(strm.SetbSync(pThis->tVars.disk.pRead, pThis->bSyncQueueFiles)); - CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); - CHKiRet(strm.SetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); - CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); - CHKiRet(strm.SetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pRead)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDel)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pReadDel, pThis->bSyncQueueFiles)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDel, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDel)); - CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(strm.SetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDeq, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDel, pThis->pszFilePrefix, pThis->lenFilePrefix)); } /* now we set (and overwrite in case of a persisted restart) some parameters which @@ -846,7 +727,8 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) * ability to read existing queue files. -- rgerhards, 2008-01-12 */ CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); - CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDeq, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDel, pThis->iMaxFileSize)); finalize_it: RETiRet; @@ -858,11 +740,13 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) DEFiRet; ASSERT(pThis != NULL); - - if (pThis->tVars.disk.pWrite != NULL) + + if(pThis->tVars.disk.pWrite != NULL) strm.Destruct(&pThis->tVars.disk.pWrite); - if (pThis->tVars.disk.pRead != NULL) - strm.Destruct(&pThis->tVars.disk.pRead); + if(pThis->tVars.disk.pReadDeq != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDeq); + if(pThis->tVars.disk.pReadDel != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDel); RETiRet; } @@ -887,23 +771,37 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) */ objDestruct(pUsr); - dbgoprint((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets\n", - nWriteCount, pThis->tVars.disk.sizeOnDisk); + DBGOPRINT((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets, EnqOnly:%d\n", + nWriteCount, pThis->tVars.disk.sizeOnDisk, pThis->bEnqOnly); finalize_it: RETiRet; } -static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) + +static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr) { DEFiRet; + CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +static rsRetVal qDelDisk(qqueue_t *pThis) +{ + obj_t *pDummyObj; /* we need to deserialize it... */ + DEFiRet; + int64 offsIn; int64 offsOut; - CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); - CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pRead, NULL, NULL)); - CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsIn)); + CHKiRet(obj.Deserialize(&pDummyObj, (uchar*) "msg", pThis->tVars.disk.pReadDel, NULL, NULL)); + objDestruct(pDummyObj); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsOut)); /* This time it is a bit tricky: we free disk space only upon file deletion. So we need * to keep track of what we have read until we get an out-offset that is lower than the @@ -915,7 +813,7 @@ static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) } else { pThis->tVars.disk.sizeOnDisk -= pThis->tVars.disk.bytesRead; pThis->tVars.disk.bytesRead = offsOut; - dbgoprint((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk); + DBGOPRINT((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk); /* awake possibly waiting enq process */ pthread_cond_signal(&pThis->notFull); /* we hold the mutex while we are in here! */ } @@ -924,6 +822,7 @@ finalize_it: RETiRet; } + /* -------------------- direct (no queueing) -------------------- */ static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { @@ -938,8 +837,11 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { + batch_t singleBatch; + batch_obj_t batchObj; DEFiRet; + //TODO: init batchObj (states _OK and new fields -- CHECK) ASSERT(pThis != NULL); /* calling the consumer is quite different here than it is from a worker thread */ @@ -947,70 +849,52 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) * mode the consumer probably has a lot to convey (which get's lost in the other modes * because they are asynchronous. But direct mode is deliberately synchronous. * rgerhards, 2008-02-12 + * We use our knowledge about the batch_t structure below, but without that, we + * pay a too-large performance toll... -- rgerhards, 2009-04-22 */ - iRet = pThis->pConsumer(pThis->pUsr, pUsr); + memset(&batchObj, 0, sizeof(batch_obj_t)); + memset(&singleBatch, 0, sizeof(batch_t)); + batchObj.state = BATCH_STATE_RDY; + batchObj.pUsrp = (obj_t*) pUsr; + batchObj.bFilterOK = 1; + singleBatch.nElem = 1; /* there always is only one in direct mode */ + singleBatch.pElem = &batchObj; + iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate); + objDestruct(pUsr); RETiRet; } -static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) -{ - return RS_RET_OK; -} - - -/* --------------- end type-specific handlers -------------------- */ - - -/* unget a user pointer that has been dequeued. This functionality is especially important - * for consumer cancel cleanup handlers. To support it, a short list of ungotten user pointers - * is maintened in memory. - * rgerhards, 2008-01-20 +/* "enqueue" a batch in direct mode. This is a shortcut which saves all the overhead + * otherwise incured. -- rgerhards, ~2010-06-23 */ -static rsRetVal -qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) +rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch) { DEFiRet; - DEFVARS_mutexProtection; - ISOBJ_TYPE_assert(pThis, qqueue); - ISOBJ_assert(pUsr); /* TODO: we aborted right at this place at least 3 times -- race? 2008-02-28, -03-10, -03-15 - The second time I noticed it the queue was in destruction with NO worker threads - running. The pUsr ptr was totally off and provided no clue what it may be pointing - at (except that it looked like the static data pool). Both times, the abort happend - inside an action queue */ + ASSERT(pThis != NULL); - dbgoprint((obj_t*) pThis, "ungetting user object %s\n", obj.GetName(pUsr)); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); - iRet = qqueueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); - ++pThis->iUngottenObjs; /* indicate one more */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + /* calling the consumer is quite different here than it is from a worker thread */ + /* we need to provide the consumer's return value back to the caller because in direct + * mode the consumer probably has a lot to convey (which get's lost in the other modes + * because they are asynchronous. But direct mode is deliberately synchronous. + * rgerhards, 2008-02-12 + * We use our knowledge about the batch_t structure below, but without that, we + * pay a too-large performance toll... -- rgerhards, 2009-04-22 + */ + iRet = pThis->pConsumer(pThis->pUsr, pBatch, &pThis->bShutdownImmediate); RETiRet; } -/* dequeues a user pointer from the ungotten queue. Pointers from there should always be - * dequeued first. - * - * This function must only be called when the mutex is locked! - * - * rgerhards, 2008-01-29 - */ -static rsRetVal -qqueueGetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) +static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) { - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(ppUsr != NULL); + return RS_RET_OK; +} - iRet = qqueueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); - --pThis->iUngottenObjs; /* indicate one less */ - dbgoprint((obj_t*) pThis, "dequeued ungotten user object %s\n", obj.GetName(*ppUsr)); - RETiRet; -} +/* --------------- end type-specific handlers -------------------- */ /* generic code to add a queue entry @@ -1029,7 +913,8 @@ qqueueAdd(qqueue_t *pThis, void *pUsr) if(pThis->qType != QUEUETYPE_DIRECT) { ATOMIC_INC(&pThis->iQueueSize, &pThis->mutQueueSize); - dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize); + DBGOPRINT((obj_t*) pThis, "entry added, size now log %d, phys %d entries\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); } finalize_it: @@ -1037,12 +922,10 @@ finalize_it: } -/* generic code to remove a queue entry - * rgerhards, 2008-01-29: we must first see if there is any object in the - * ungotten list and, if so, dequeue it first. +/* generic code to dequeue a queue entry */ static rsRetVal -qqueueDel(qqueue_t *pThis, void *pUsr) +qqueueDeq(qqueue_t *pThis, void **ppUsr) { DEFiRet; @@ -1053,224 +936,249 @@ qqueueDel(qqueue_t *pThis, void *pUsr) * If we decrement, however, we may lose a message. But that is better than * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ - if(pThis->iUngottenObjs > 0) { - iRet = qqueueGetUngottenObj(pThis, (obj_t**) pUsr); - } else { - iRet = pThis->qDel(pThis, pUsr); - ATOMIC_DEC(&pThis->iQueueSize, &pThis->mutQueueSize); - } + iRet = pThis->qDeq(pThis, ppUsr); + ATOMIC_INC(&pThis->nLogDeq, &pThis->mutLogDeq); - dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n", - iRet, pThis->iQueueSize); +// DBGOPRINT((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", +// getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } -/* This function shuts down all worker threads and waits until they - * have terminated. If they timeout, they are cancelled. Parameters have been set - * before this function is called so that DA queues will be fully persisted to - * disk (if configured to do so). - * rgerhards, 2008-01-24 - * Please note that this function shuts down BOTH the parent AND the child queue - * in DA case. This is necessary because their timeouts are tightly coupled. Most - * importantly, the timeouts would be applied twice (or logic be extremely - * complex) if each would have its own shutdown. The function does not self check - * this condition - the caller must make sure it is not called with a parent. +/* Try to shut down regular and DA queue workers, within the queue timeout + * period. That means processing continues as usual. This is the expected + * usual case, where during shutdown those messages remaining are being + * processed. At this point, it is acceptable that the queue can not be + * fully depleted, that case is handled in the next step. During this phase, + * we first shut down the main queue DA worker to prevent new data to arrive + * at the DA queue, and then we ask the regular workers of both the Regular + * and DA queue to try complete processing. + * rgerhards, 2009-10-14 */ -static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) +static inline rsRetVal +tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; struct timespec tTimeout; rsRetVal iRetLocal; + DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ - dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); + if(pThis->bIsDA) { + /* We need to lock the mutex, as otherwise we may have a race that prevents + * us from awaking the DA worker. */ + d_pthread_mutex_lock(pThis->mut); - /* we reduce the low water mark in any case. This is not absolutely necessary, but - * it is useful because we enable DA mode at several spots below and so we do not need - * to think about the low water mark each time. - */ - pThis->iHighWtrMrk = 1; /* if we do not do this, the DA queue will not stop! */ - pThis->iLowWtrMrk = 0; + /* tell regular queue DA worker to stop shuffling messages to DA queue... */ + DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode for DA worker\n"); + pThis->pqDA->bEnqOnly = 1; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE); + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); + DBGOPRINT((obj_t*) pThis, "awoke DA worker, told it to shut down.\n"); - /* first try to shutdown the queue within the regular shutdown period */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - if(pThis->bRunsDA) { - /* We may have waited on the low water mark. As it may have changed, we - * see if we reactivate the worker. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); - } + /* also tell the DA queue worker to shut down, so that it already knows... */ + wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN); + wtpAdviseMaxWorkers(pThis->pqDA->pWtpReg, 1); /* awake its lone worker */ + DBGOPRINT((obj_t*) pThis, "awoke DA queue regular worker, told it to shut down when done.\n"); + + d_pthread_mutex_unlock(pThis->mut); } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - /* Now wait for the queue's workers to shut down. Note that we run into the code even if we just found - * out there are no active workers - that doesn't matter: the wtp knows about that and so will - * return immediately. - * We do not yet care about the DA worker - that will be handled down later in the process. - * Note that we must not request shutdown right now - that may introduce a race: if the regular queue - * still runs DA assisted and the DA worker gets scheduled first, it will terminate itself (if the DA - * queue happens to be empty at that instant). Then the regular worker enqueues messages, what will lead - * to a restart of the worker. Of course, everything will continue to run, but in a bit sub-optimal way - * (from a performance point of view). So we don't do anything right now. The DA queue will continue to - * process messages and shutdown itself in any case if there is nothing to do. So we don't loose anything - * by not requesting shutdown now. - * rgerhards, 2008-01-25 - */ + /* first calculate absolute timeout - we need the absolute value here, because we need to coordinate * shutdown of both the regular and DA queue on *the same* timeout. */ timeoutComp(&tTimeout, pThis->toQShutdown); - dbgoprint((obj_t*) pThis, "trying shutdown of regular workers\n"); + DBGOPRINT((obj_t*) pThis, "trying shutdown of regular workers\n"); iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN, &tTimeout); if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n"); + DBGOPRINT((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n"); } else { - /* OK, the regular queue is now shut down. So we can now wait for the DA queue (if running DA) */ - dbgoprint((obj_t*) pThis, "regular queue workers shut down.\n"); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(pThis->bRunsDA) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", - qqueueGetID(pThis->pqDA)); - /* we use the same absolute timeout as above, so we do not use more than the configured - * timeout interval! - */ - dbgoprint((obj_t*) pThis, "trying shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "shutdown timed out on DA queue (this is OK)\n"); - } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - } + DBGOPRINT((obj_t*) pThis, "regular queue workers shut down.\n"); } - /* when we reach this point, both queues are either empty or the regular queue shutdown timeout - * has expired. Now we need to check if we are configured to not loose messages. If so, we need - * to persist the queue to disk (this is only possible if the queue is DA-enabled). We must also - * set the primary queue to SHUTDOWN_IMMEDIATE, as it shall now terminate as soon as its consumer - * is done. This is especially important as we otherwise may interfere with queue order while the - * DA consumer is running. -- rgerhards, 2008-01-27 - * Note: there was a note that we should not wait eternally on the DA worker if we run in - * enqueue-only note. I have reviewed the code and think there is no need for this check. Howerver, - * I'd like to keep this note in here should we happen to run into some related trouble. - * rgerhards, 2008-01-28 - */ - wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); /* set primary queue to shutdown only */ - - /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ - if(pThis->bRunsDA) - qqueueWaitDAModeInitialized(pThis); - - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - /* optimize parameters for shutdown of DA-enabled queues */ - if(pThis->bIsDA && qqueueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { - /* switch to enqueue-only mode so that no more actions happen */ - if(pThis->bRunsDA == 0) { - qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ + /* OK, the worker for the regular queue is processed, on the the DA queue regular worker. */ + if(pThis->pqDA != NULL) { + DBGOPRINT((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", + qqueueGetID(pThis->pqDA)); + /* we use the same absolute timeout as above, so we do not use more than the configured + * timeout interval! + */ + DBGOPRINT((obj_t*) pThis, "trying shutdown of regular worker of DA queue\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "shutdown timed out on DA queue worker (this is OK)\n"); } else { - /* TODO: RACE: we may reach this point when the DA worker has been initialized (state 1) - * but is not yet running (state 2). In this case, pThis->pqDA is NULL! rgerhards, 2008-02-27 - */ - qqueueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ - } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - /* make sure we do not timeout before we are done */ - dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, eternal timeout set\n"); - timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); - /* and run the primary queue's DA worker to drain the queue */ - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " - "continuing, but results are unpredictable\n", iRetLocal); + DBGOPRINT((obj_t*) pThis, "DA queue worker shut down.\n"); } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); } - /* now the primary queue is either empty, persisted to disk - or set to loose messages. So we - * can now request immediate shutdown of any remaining workers. Note that if bSaveOnShutdown was set, - * the queue is now empty. If regular workers are still running, and try to pull the next message, - * they will automatically terminate as there no longer is any message left to process. - */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { - timeoutComp(&tTimeout, pThis->toActShutdown); - if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of regular workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } - /* we need to re-aquire the mutex for the next check in this case! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ + RETiRet; +} + + +/* Try to shut down regular and DA queue workers, within the action timeout + * period. This aborts processing, but at the end of the current action, in + * a well-defined manner. During this phase, we terminate all three worker + * pools, including the regular queue DA worker if it not yet has terminated. + * Not finishing processing all messages is OK (and expected) at this stage + * (they may be preserved later, depending * on bSaveOnShutdown setting). + * rgerhards, 2009-10-14 + */ +static rsRetVal +tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) +{ + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + +RUNLOG_STR("trying to shutdown workers within Action Timeout"); + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + /* instruct workers to finish ASAP, even if still work exists */ + DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode\n"); + pThis->bEnqOnly = 1; + pThis->bShutdownImmediate = 1; + /* now DA queue */ + if(pThis->bIsDA) { + pThis->pqDA->bEnqOnly = 1; + pThis->pqDA->bShutdownImmediate = 1; + } + +// TODO: make sure we have at minimum a 10ms timeout - workers deserve a chance... + /* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */ + timeoutComp(&tTimeout, pThis->toActShutdown); + DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of regular workers (if any)\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " + "triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " + "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); + } + + if(pThis->pqDA != NULL) { + /* and now the same for the DA queue */ + DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of DA queue workers\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable " + "and triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA " + "queue in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); } - if(pThis->bRunsDA && wtpGetCurNumWrkr(pThis->pWtpDA, LOCK_MUTEX) > 0) { - /* and now the same for the DA queue */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } + + /* and now we need to terminate the DA worker itself. We always grant it a 100ms timeout, + * which should be sufficient and usually not be required (it is expected to have finished + * long before while we were processing the queue timeout in shutdown phase 1). + * rgerhards, 2009-10-14 + */ + timeoutComp(&tTimeout, 100); + DBGOPRINT((obj_t*) pThis, "trying regular shutdown of main queue DA worker pool\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + DBGOPRINT((obj_t*) pThis, "shutdown timed out on main queue DA worker pool " + "(this is not good, but probably OK)\n"); } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + DBGOPRINT((obj_t*) pThis, "main queue DA worker pool shut down.\n"); } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); } + RETiRet; +} + + +/* This function cancels all remaining regular workers for both the main and the DA + * queue. + * rgerhards, 2009-05-29 + */ +static rsRetVal +cancelWorkers(qqueue_t *pThis) +{ + rsRetVal iRetLocal; + DEFiRet; + /* Now queue workers should have terminated. If not, we need to cancel them as we have applied * all timeout setting. If any worker in any queue still executes, its consumer is possibly - * long-running and cancelling is the only way to get rid of it. Note that the - * cancellation handler will probably re-queue a user pointer, so the queue's enqueue - * function is still needed (what is no problem as we do not yet destroy the queue - but I - * thought it's a good idea to mention that fact). -- rgerhards, 2008-01-25 + * long-running and cancelling is the only way to get rid of it. */ - dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n"); + DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n"); iRetLocal = wtpCancelAll(pThis->pWtpReg); /* returns immediately if all threads already have terminated */ if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker " + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker " "threads, continuing, but results are unpredictable\n", iRetLocal); } - - /* TODO: think: do we really need to do this here? Can't it happen on DA queue destruction? If we - * disable it, we get an assertion... I think this is OK, as we need to have a certain order and - * canceling the DA workers here ensures that order. But in any instant, we may have a look at this - * code after we have reaced the milestone. -- rgerhards, 2008-01-27 - */ /* ... and now the DA queue, if it exists (should always be after the primary one) */ if(pThis->pqDA != NULL) { - dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n"); + DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n"); iRetLocal = wtpCancelAll(pThis->pqDA->pWtpReg); /* returns immediately if all threads already have terminated */ if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker " + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker " "threads, continuing, but results are unpredictable\n", iRetLocal); } + + /* finally, we cancel the main queue's DA worker pool, if it still is running. It may be + * restarted later to persist the queue. But we stop it, because otherwise we get into + * big trouble when resetting the logical dequeue pointer. This operation can only be + * done when *no* worker is running. So time for a shutdown... -- rgerhards, 2009-05-28 + */ + DBGOPRINT((obj_t*) pThis, "checking to see if main queue DA worker pool needs to be cancelled\n"); + iRetLocal = wtpCancelAll(pThis->pWtpDA); /* returns immediately if all threads already have terminated */ + } + + RETiRet; +} + + +/* This function shuts down all worker threads and waits until they + * have terminated. If they timeout, they are cancelled. + * rgerhards, 2008-01-24 + * Please note that this function shuts down BOTH the parent AND the child queue + * in DA case. This is necessary because their timeouts are tightly coupled. Most + * importantly, the timeouts would be applied twice (or logic be extremely + * complex) if each would have its own shutdown. The function does not self check + * this condition - the caller must make sure it is not called with a parent. + * rgerhards, 2009-05-26: we do NO longer persist the queue here if bSaveOnShutdown + * is set. This must be handled by the caller. Not doing that cleans up the queue + * shutdown considerably. Also, older engines had a potential hang condition when + * the DA queue was already started and the DA worker configured for infinite + * retries and the action was during retry processing. This was a design issue, + * which is solved as of now. Note that the shutdown now may take a little bit + * longer, because we no longer can persist the queue in parallel to waiting + * on worker timeouts. + */ +static rsRetVal +ShutdownWorkers(qqueue_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + DBGOPRINT((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); + + CHKiRet(tryShutdownWorkersWithinQueueTimeout(pThis)); + + if(getPhysicalQueueSize(pThis) > 0) { + CHKiRet(tryShutdownWorkersWithinActionTimeout(pThis)); } + CHKiRet(cancelWorkers(pThis)); + /* ... finally ... all worker threads have terminated :-) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers - * may still be running. + * may still be running. Note that the main queue's DA worker may still be running. */ - dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", qqueueGetOverallQueueSize(pThis)); + DBGOPRINT((obj_t*) pThis, "worker threads terminated, remaining queue size log %d, phys %d.\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); +finalize_it: RETiRet; } @@ -1282,7 +1190,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * to modify some parameters before the queue is actually started. */ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)) + int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*,int*)) { DEFiRet; qqueue_t *pThis; @@ -1291,9 +1199,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread ASSERT(pConsumer != NULL); ASSERT(iWorkerThreads >= 0); - if((pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))); /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); @@ -1304,13 +1210,15 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->iFullDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 3; /* default 97% */ pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */ - pThis->lenSpoolDir = strlen((char*)pThis->pszSpoolDir); + pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; + pThis->nLogDeq = 0; pThis->iMaxQueueSize = iMaxQueueSize; pThis->pConsumer = pConsumer; pThis->iNumWorkerThreads = iWorkerThreads; pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */ + pThis->iDeqBatchSize = 8; /* conservative default, should still provide good performance */ pThis->pszFilePrefix = NULL; pThis->qType = qType; @@ -1321,19 +1229,25 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qConstruct = qConstructFixedArray; pThis->qDestruct = qDestructFixedArray; pThis->qAdd = qAddFixedArray; + pThis->qDeq = qDeqFixedArray; pThis->qDel = qDelFixedArray; + pThis->MultiEnq = qqueueMultiEnqObjNonDirect; break; case QUEUETYPE_LINKEDLIST: pThis->qConstruct = qConstructLinkedList; pThis->qDestruct = qDestructLinkedList; pThis->qAdd = qAddLinkedList; - pThis->qDel = (rsRetVal (*)(qqueue_t*,void**)) qDelLinkedList; + pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList; + pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList; + pThis->MultiEnq = qqueueMultiEnqObjNonDirect; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; pThis->qDestruct = qDestructDisk; pThis->qAdd = qAddDisk; + pThis->qDeq = qDeqDisk; pThis->qDel = qDelDisk; + pThis->MultiEnq = qqueueMultiEnqObjNonDirect; /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ break; @@ -1342,10 +1256,12 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qDestruct = qDestructDirect; pThis->qAdd = qAddDirect; pThis->qDel = qDelDirect; + pThis->MultiEnq = qqueueMultiEnqObjDirect; break; } INIT_ATOMIC_HELPER_MUT(pThis->mutQueueSize); + INIT_ATOMIC_HELPER_MUT(pThis->mutLogDeq); finalize_it: OBJCONSTRUCT_CHECK_SUCCESS_AND_CLEANUP @@ -1353,40 +1269,10 @@ finalize_it: } -/* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. - * Params: - * arg1 - user pointer (in this case a qqueue_t) - * arg2 - user data pointer (in this case a queue data element, any object [queue's pUsr ptr!]) - * Note that arg2 may be NULL, in which case no dequeued but unprocessed pUsr exists! - * rgerhards, 2008-01-16 - */ -static rsRetVal -qqueueConsumerCancelCleanup(void *arg1, void *arg2) -{ - DEFiRet; - - qqueue_t *pThis = (qqueue_t*) arg1; - obj_t *pUsr = (obj_t*) arg2; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pUsr != NULL) { - /* make sure the data element is not lost */ - dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n"); - CHKiRet(qqueueUngetObj(pThis, pUsr, LOCK_MUTEX)); - } - -finalize_it: - RETiRet; -} - - - /* This function checks if the provided message shall be discarded and does so, if needed. * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to * provide real-time creation of spool files. - * Note: cached copies of iQueueSize and bRunsDA are provided so that no mutex locks are required. + * Note: cached copies of iQueueSize is provided so that no mutex locks are required. * The caller must have obtained them while the mutex was locked. Of course, these values may no * longer be current, but that is OK for the discard check. At worst, the message is either processed * or discarded when it should not have been. As discarding is in itself somewhat racy and erratic, @@ -1396,7 +1282,7 @@ finalize_it: * the return state! * rgerhards, 2008-01-24 */ -static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) +static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, void *pUsr) { DEFiRet; rsRetVal iRetLocal; @@ -1405,15 +1291,15 @@ static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, voi ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); - if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk && bRunsDA == 0) { + if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk) { iRetLocal = objGetSeverity(pUsr, &iSeverity); if(iRetLocal == RS_RET_OK && iSeverity >= pThis->iDiscardSeverity) { - dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", + DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", iQueueSize, iSeverity); objDestruct(pUsr); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } else { - dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg " + DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg " "(iRet: %d, severity %d)\n", iQueueSize, iRetLocal, iSeverity); } } @@ -1423,38 +1309,191 @@ finalize_it: } -/* dequeue the queued object for the queue consumers. - * rgerhards, 2008-10-21 +/* Finally remove n elements from the queue store. */ -static rsRetVal -qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +static inline rsRetVal +DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) { + int i; DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + /* now send delete request to storage driver */ + for(i = 0 ; i < nElem ; ++i) { + pThis->qDel(pThis); + } + + /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ + ATOMIC_SUB(&pThis->iQueueSize, nElem, &pThis->mutQueueSize); + ATOMIC_SUB(&pThis->nLogDeq, nElem, &pThis->mutLogDeq); +dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + ++pThis->deqIDDel; /* one more batch dequeued */ + + RETiRet; +} + + +/* remove messages from the physical queue store that are fully processed. This is + * controlled via the to-delete list. + */ +static inline rsRetVal +DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) +{ + toDeleteLst_t *pTdl; + qDeqID deqIDDel; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + pTdl = tdlPeek(pThis); /* get current head element */ + if(pTdl == NULL) { /* to-delete list empty */ + DoDeleteBatchFromQStore(pThis, pBatch->nElem); + } else if(pBatch->deqID == pThis->deqIDDel) { + deqIDDel = pThis->deqIDDel; + pTdl = tdlPeek(pThis); + while(pTdl != NULL && deqIDDel == pTdl->deqID) { + DoDeleteBatchFromQStore(pThis, pTdl->nElemDeq); + tdlPop(pThis); + ++deqIDDel; + pTdl = tdlPeek(pThis); + } + /* old entries deleted, now delete current ones... */ + DoDeleteBatchFromQStore(pThis, pBatch->nElem); + } else { + /* can not delete, insert into to-delete list */ + dbgprintf("not at head of to-delete list, enqueue %d\n", (int) pBatch->deqID); + CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem)); + } + +finalize_it: + RETiRet; +} + + +/* Delete a batch of processed user objects from the queue, which includes + * destructing the objects themself. Any entries not marked as finally + * processed are enqueued again. The new enqueue is necessary because we have a + * rgerhards, 2009-05-13 + */ +static inline rsRetVal +DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) +{ + int i; void *pUsr; + int nEnqueued = 0; + rsRetVal localRet; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + for(i = 0 ; i < pBatch->nElem ; ++i) { + pUsr = pBatch->pElem[i].pUsrp; + if( pBatch->pElem[i].state == BATCH_STATE_RDY + || pBatch->pElem[i].state == BATCH_STATE_SUB) { +dbgprintf("XXX: DeleteProcessedBatch re-enqueue %d of %d, state %d\n", i, pBatch->nElem, pBatch->pElem[i].state); + localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*) pUsr)); + ++nEnqueued; + if(localRet != RS_RET_OK) { + DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet); + } + } + objDestruct(pUsr); + } + + dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued); + + if(nEnqueued > 0) + qqueueChkPersist(pThis, nEnqueued); + + iRet = DeleteBatchFromQStore(pThis, pBatch); + + pBatch->nElem = pBatch->nElemDeq = 0; /* reset batch */ // TODO: more fine init, new fields! 2010-06-14 + + RETiRet; +} + + +/* dequeue as many user pointers as are available, until we hit the configured + * upper limit of pointers. Note that this function also deletes all processed + * objects from the previous batch. However, it is perfectly valid that the + * previous batch contained NO objects at all. For example, this happens + * immediately after system startup or when a queue was exhausted and the queue + * worker needed to wait for new data. + * This must only be called when the queue mutex is LOOKED, otherwise serious + * malfunction will happen. + */ +static inline rsRetVal +DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSize) +{ + int nDequeued; + int nDiscarded; + int nDeleted; int iQueueSize; - int bRunsDA; /* cache for early mutex release */ - - /* dequeue element (still protected from mutex) */ - iRet = qqueueDel(pThis, &pUsr); - qqueueChkPersist(pThis); - iQueueSize = qqueueGetOverallQueueSize(pThis); /* cache this for after mutex release */ - bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ - - /* We now need to save the user pointer for the cancel cleanup handler, BUT ONLY - * if we could successfully obtain a user pointer. Otherwise, we would bring the - * cancel cleanup handler into big troubles (and we did ;)). Note that we can - * NOT set the variable further below, as this may lead to an object leak. We - * may get cancelled before we reach that part of the code, so the only - * solution is to do it here. -- rgerhards, 2008-02-27 - */ - if(iRet == RS_RET_OK) { - pWti->pUsrp = pUsr; + void *pUsr; + rsRetVal localRet; + DEFiRet; + + nDeleted = pWti->batch.nElemDeq; + DeleteProcessedBatch(pThis, &pWti->batch); + + nDequeued = nDiscarded = 0; + while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { + CHKiRet(qqueueDeq(pThis, &pUsr)); + + /* check if we should discard this element */ + localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr); + if(localRet == RS_RET_QUEUE_FULL) { + ++nDiscarded; + continue; + } else if(localRet != RS_RET_OK) { + ABORT_FINALIZE(localRet); + } + + /* all well, use this element */ + pWti->batch.pElem[nDequeued].pUsrp = pUsr; + pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY; + pWti->batch.pElem[nDequeued].bFilterOK = 1; // TODO: think again if we can handle that with more performance + ++nDequeued; } + /* it is sufficient to persist only when the bulk of work is done */ + qqueueChkPersist(pThis, nDequeued+nDiscarded+nDeleted); + + pWti->batch.nElem = nDequeued; + pWti->batch.nElemDeq = nDequeued + nDiscarded; + pWti->batch.deqID = getNextDeqID(pThis); + *piRemainingQueueSize = iQueueSize; + +finalize_it: + RETiRet; +} + + +/* dequeue the queued object for the queue consumers. + * rgerhards, 2008-10-21 + * I made a radical change - we now dequeue multiple elements, and store these objects in + * an array of user pointers. We expect that this increases performance. + * rgerhards, 2009-04-22 + */ +static rsRetVal +DequeueConsumable(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + int iQueueSize = 0; /* keep the compiler happy... */ + + /* dequeue element batch (still protected from mutex) */ + iRet = DequeueConsumableElements(pThis, pWti, &iQueueSize); + /* awake some flow-controlled sources if we can do this right now */ /* TODO: this could be done better from a performance point of view -- do it only if * we have someone waiting for the condition (or only when we hit the watermark right * on the nail [exact value]) -- rgerhards, 2008-03-14 + * now that we dequeue batches of pointers, this is much less an issue... + * rgerhards, 2009-04-22 */ if(iQueueSize < pThis->iFullDlyMrk / 2) { pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk); @@ -1464,37 +1503,15 @@ qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk); } - /* rgerhards, 2008-09-30: I reversed the order of cond_signal und mutex_unlock - * as of the pthreads recommendation on predictable scheduling behaviour. I don't see - * any problems caused by this, but I add this comment in case some will be seen - * in the next time. - */ + // TODO: MULTI: check physical queue size? pthread_cond_signal(&pThis->notFull); - d_pthread_mutex_unlock(pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); /* WE ARE NO LONGER PROTECTED BY THE MUTEX */ - /* do actual processing (the lengthy part, runs in parallel) - * If we had a problem while dequeing, we do not call the consumer, - * but we otherwise ignore it. This is in the hopes that it will be - * self-healing. However, this is really not a good thing. - * rgerhards, 2008-01-03 - */ - if(iRet != RS_RET_OK) - FINALIZE; - - /* we are running in normal, non-disk-assisted mode do a quick check if we need to drain the queue. - * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to - * provide real-time creation of spool files. - * Note: It is OK to use the cached iQueueSize here, because it does not hurt if it is slightly wrong. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); - -finalize_it: if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) { - dbgoprint((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things " + DBGOPRINT((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things " "may happen\n", iRet); } + RETiRet; } @@ -1537,7 +1554,7 @@ finalize_it: * but you get the idea from the code above. */ static rsRetVal -qqueueRateLimiter(qqueue_t *pThis) +RateLimiter(qqueue_t *pThis) { DEFiRet; int iDelay; @@ -1550,7 +1567,7 @@ qqueueRateLimiter(qqueue_t *pThis) iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ /* time calls are expensive, so only do them when needed */ - time(&tCurr); + datetime.GetTime(&tCurr); localtime_r(&tCurr, &m); iHrCurr = m.tm_hour; @@ -1586,7 +1603,7 @@ qqueueRateLimiter(qqueue_t *pThis) } if(iDelay > 0) { - dbgoprint((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); + DBGOPRINT((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); srSleep(iDelay, 0); } @@ -1594,37 +1611,105 @@ qqueueRateLimiter(qqueue_t *pThis) } +/* This dequeues the next batch. Note that this function must not be + * cancelled, else it will leave back an inconsistent state. + * rgerhards, 2009-05-20 + */ +static inline rsRetVal +DequeueForConsumer(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); + + CHKiRet(DequeueConsumable(pThis, pWti)); + + if(pWti->batch.nElem == 0) + ABORT_FINALIZE(RS_RET_IDLE); + + +finalize_it: + RETiRet; +} + + +/* This is called when a batch is processed and the worker does not + * ask for another batch (e.g. because it is to be terminated) + * Note that we must not be terminated while we delete a processed + * batch. Otherwise, we may not complete it, and then the cancel + * handler also tries to delete the batch. But then it finds some of + * the messages already destructed. This was a bug we have seen, especially + * with disk mode, where a delete takes rather long. Anyhow, the coneptual + * problem exists in all queue modes. + * rgerhards, 2009-05-27 + */ +static rsRetVal +batchProcessed(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); + + int iCancelStateSave; + /* at this spot, we must not be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + DeleteProcessedBatch(pThis, &pWti->batch); + qqueueChkPersist(pThis, pWti->batch.nElemDeq); + pthread_setcancelstate(iCancelStateSave, NULL); + + RETiRet; +} + /* This is the queue consumer in the regular (non-DA) case. It is * protected by the queue mutex, but MUST release it as soon as possible. * rgerhards, 2008-01-21 */ static rsRetVal -qqueueConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerReg(qqueue_t *pThis, wti_t *pWti) { + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->pUsrp)); + CHKiRet(DequeueForConsumer(pThis, pWti)); + + /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ + d_pthread_mutex_unlock(pThis->mut); + + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + + CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch, &pThis->bShutdownImmediate)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 */ +//TODO: MULTIQUEUE: the following setting is no longer correct - need to think about how to do that... if(pThis->iDeqSlowdown) { - dbgoprint((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n", + DBGOPRINT((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n", pThis->iDeqSlowdown); srSleep(pThis->iDeqSlowdown / 1000000, pThis->iDeqSlowdown % 1000000); } + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + + /* now we are done, but need to re-aquire the mutex */ + d_pthread_mutex_lock(pThis->mut); + finalize_it: + dbgprintf("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } -/* This is a special consumer to feed the disk-queue in disk-assited mode. +/* This is a special consumer to feed the disk-queue in disk-assisted mode. * When active, our own queue more or less acts as a memory buffer to the disk. * So this consumer just needs to drain the memory queue and submit entries * to the disk queue. The disk queue will then call the actual consumer from @@ -1634,151 +1719,92 @@ finalize_it: * rgerhards, 2008-01-14 */ static rsRetVal -qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerDA(qqueue_t *pThis, wti_t *pWti) { + int i; + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); + CHKiRet(DequeueForConsumer(pThis, pWti)); + + /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ + d_pthread_mutex_unlock(pThis->mut); + + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + + /* iterate over returned results and enqueue them in DA queue */ + for(i = 0 ; i < pWti->batch.nElem && !pThis->bShutdownImmediate ; i++) { + /* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs + * the message. So far, we simply assume we always have msg_t, what currently is always the case. + * rgerhards, 2009-05-28 + */ + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp)))); + pWti->batch.pElem[i].state = BATCH_STATE_COMM; /* commited to other queue! */ + } + + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + + /* now we are done, but need to re-aquire the mutex */ + d_pthread_mutex_lock(pThis->mut); finalize_it: - dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); + DBGOPRINT((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); RETiRet; } /* must only be called when the queue mutex is locked, else results * are not stable! - * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the DA worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue */ -static int +static rsRetVal qqueueChkStopWrkrDA(qqueue_t *pThis) { - /* if our queue is in destruction, we drain to the DA queue and so we shall not terminate - * until we have done so. - */ - int bStopWrkr; - - BEGINfunc + DEFiRet; if(pThis->bEnqOnly) { - bStopWrkr = 1; - } else { - if(pThis->bRunsDA == 2) { - ASSERT(pThis->pqDA != NULL); - if( pThis->pqDA->bEnqOnly - && pThis->pqDA->sizeOnDiskMax > 0 - && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { - /* this queue can never grow, so we can give up... */ - bStopWrkr = 1; - } else if(qqueueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { - bStopWrkr = 1; - } else { - bStopWrkr = 0; - } - } else { - bStopWrkr = 1; - } + iRet = RS_RET_TERMINATE_WHEN_IDLE; } - ENDfunc - return bStopWrkr; + RETiRet; } /* must only be called when the queue mutex is locked, else results * are not stable! * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the regular worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue - */ -static int -qqueueChkStopWrkrReg(qqueue_t *pThis) -{ - return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0); -} - - -/* must only be called when the queue mutex is locked, else results - * are not stable! DA queue version - */ -static int -qqueueIsIdleDA(qqueue_t *pThis) -{ - /* remember: iQueueSize is the DA queue size, not the main queue! */ - /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); -} -/* must only be called when the queue mutex is locked, else results - * are not stable! Regular queue version - */ -static int -qqueueIsIdleReg(qqueue_t *pThis) -{ -#if 0 /* enable for performance testing */ - int ret; - ret = qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); - if(ret) fprintf(stderr, "queue is idle\n"); - return ret; -#else - /* regular code! */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); -#endif -} - - -/* This function is called when a worker thread for the regular queue is shut down. - * If we are the primary queue, this is not really interesting to us. If, however, - * we are the DA (child) queue, that means the DA queue is empty. In that case, we - * need to signal the parent queue's DA worker, so that it can terminate DA mode. - * rgerhards, 2008-01-26 - * rgerhards, 2008-02-27: HOWEVER, in a shutdown condition, it may be that the parent's worker thread pool - * has already been terminated and destructed. This *is* a legal condition and happens - * from time to time in practice. So we need to signal only if there still is a - * parent DA worker queue. Please keep in mind that the the parent's DA worker - * pool is DIFFERENT from our (DA queue) regular worker pool. So when the parent's - * pWtpDA is destructed, there can still be some of our (DAq/wtp) threads be running. - * I am telling this, because I, too, always get confused by those... + * we can terminate. Version for the regular worker thread. */ static rsRetVal -qqueueRegOnWrkrShutdown(qqueue_t *pThis) +ChkStopWrkrReg(qqueue_t *pThis) { DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pqParent != NULL) { - pThis->pqParent->bChildIsDone = 1; /* indicate we are done */ - if(pThis->pqParent->pWtpDA != NULL) { /* see comment in function header from 2008-02-27 */ - wtpAdviseMaxWorkers(pThis->pqParent->pWtpDA, 1); /* reactivate DA worker (always 1) */ - } + if(pThis->bEnqOnly) { + iRet = RS_RET_TERMINATE_NOW; + } else if(pThis->pqParent != NULL) { + iRet = RS_RET_TERMINATE_WHEN_IDLE; } RETiRet; } -/* The following function is called when a regular queue worker starts up. We need this - * hook to indicate in the parent queue (if we are a child) that we are not done yet. +/* return the configured "deq max at once" interval + * rgerhards, 2009-04-22 */ static rsRetVal -qqueueRegOnWrkrStartup(qqueue_t *pThis) +GetDeqBatchSize(qqueue_t *pThis, int *pVal) { DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pqParent != NULL) { - pThis->pqParent->bChildIsDone = 0; - } - + assert(pVal != NULL); + *pVal = pThis->iDeqBatchSize; +if(pThis->pqParent != NULL) // TODO: check why we actually do this! + *pVal = 16; RETiRet; } @@ -1786,11 +1812,10 @@ qqueueRegOnWrkrStartup(qqueue_t *pThis) /* start up the queue - it must have been constructed and parameters defined * before. */ -rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ +rsRetVal +qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; - rsRetVal iRetLocal; - int bInitialized = 0; /* is queue already initialized? */ uchar pszBuf[64]; size_t lenBuf; @@ -1804,11 +1829,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ * influenced by properties which might have been set after queueConstruct () */ if(pThis->pqParent == NULL) { - pThis->mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + pThis->mut = (pthread_mutex_t *) MALLOC (sizeof (pthread_mutex_t)); pthread_mutex_init(pThis->mut, NULL); } else { /* child queue, we need to use parent's mutex */ - dbgoprint((obj_t*) pThis, "I am a child\n"); + DBGOPRINT((obj_t*) pThis, "I am a child\n"); pThis->mut = pThis->pqParent->mut; } @@ -1822,28 +1847,26 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* call type-specific constructor */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ - dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d, " - "full delay %d, light delay %d starting\n", + DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, " + "full delay %d, light delay %d, deq batch size %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, - qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, - pThis->iFullDlyMrk, pThis->iLightDlyMrk); + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), + pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, + pThis->iDeqBatchSize); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ - /* create worker thread pools for regular operation. The DA pool is created on an as-needed - * basis, which potentially means never under most circumstances. + /* create worker thread pools for regular and DA operation. */ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpReg)); CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); - CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRateLimiter)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrReg)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleReg)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerReg)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))qqueueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrStartup)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrShutdown)); + CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerReg)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads)); @@ -1851,27 +1874,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpUsr (pThis->pWtpReg, pThis)); CHKiRet(wtpConstructFinalize (pThis->pWtpReg)); - /* initialize worker thread instances */ - if(pThis->bIsDA) { - /* If we are disk-assisted, we need to check if there is a QIF file - * which we need to load. -- rgerhards, 2008-01-15 - */ - iRetLocal = qqueueHaveQIF(pThis); - if(iRetLocal == RS_RET_OK) { - dbgoprint((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n"); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ - bInitialized = 1; /* we are done */ - } else { - /* TODO: use logerror? -- rgerhards, 2008-01-16 */ - dbgoprint((obj_t*) pThis, "error %d trying to access on-disk queue files, starting without them. " - "Some data may be lost\n", iRetLocal); - } - } + /* set up DA system if we have a disk-assisted queue */ + if(pThis->bIsDA) + InitDA(pThis, LOCK_MUTEX); /* initiate DA mode */ - if(!bInitialized) { - dbgoprint((obj_t*) pThis, "queue starts up without (loading) any DA disk state (this is normal for the DA " - "queue itself!)\n"); - } + DBGOPRINT((obj_t*) pThis, "queue finished initialization\n"); /* if the queue already contains data, we need to start the correct number of worker threads. This can be * the case when a disk queue has been loaded. If we did not start it here, it would never start. @@ -1897,12 +1904,11 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) strm_t *psQIF = NULL; /* Queue Info File */ uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; - obj_t *pUsr; ASSERT(pThis != NULL); if(pThis->qType != QUEUETYPE_DISK) { - if(qqueueGetOverallQueueSize(pThis) > 0) { + if(getPhysicalQueueSize(pThis) > 0) { /* This error code is OK, but we will probably not implement this any time * The reason is that persistence happens via DA queues. But I would like to * leave the code as is, as we so have a hook in case we need one. @@ -1913,20 +1919,20 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) FINALIZE; /* if the queue is empty, we are happy and done... */ } - dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", qqueueGetOverallQueueSize(pThis)); + DBGOPRINT((obj_t*) pThis, "persisting queue to disk, %d entries...\n", getPhysicalQueueSize(pThis)); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - if((bIsCheckpoint != QUEUE_CHECKPOINT) && (qqueueGetOverallQueueSize(pThis) == 0)) { + if((bIsCheckpoint != QUEUE_CHECKPOINT) && (getPhysicalQueueSize(pThis) == 0)) { if(pThis->bNeedDelQIF) { unlink((char*)pszQIFNam); pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ - if (pThis->tVars.disk.pRead != NULL) - CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + if(pThis->tVars.disk.pReadDel != NULL) /* may be NULL if we had a startup failure! */ + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); FINALIZE; /* nothing left to do, so be happy */ } @@ -1945,31 +1951,19 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) */ CHKiRet(obj.BeginSerializePropBag(psQIF, (obj_t*) pThis)); objSerializeSCALAR(psQIF, iQueueSize, INT); - objSerializeSCALAR(psQIF, iUngottenObjs, INT); objSerializeSCALAR(psQIF, tVars.disk.sizeOnDisk, INT64); objSerializeSCALAR(psQIF, tVars.disk.bytesRead, INT64); CHKiRet(obj.EndSerialize(psQIF)); - /* now we must persist all objects on the ungotten queue - they can not go to - * to the regular files. -- rgerhards, 2008-01-29 - */ - while(pThis->iUngottenObjs > 0) { - CHKiRet(qqueueGetUngottenObj(pThis, &pUsr)); - CHKiRet((objSerialize(pUsr))(pUsr, psQIF)); - objDestruct(pUsr); - } - /* now persist the stream info */ - if (pThis->tVars.disk.pWrite != NULL) - CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); - if (pThis->tVars.disk.pRead != NULL) - CHKiRet(strm.Serialize(pThis->tVars.disk.pRead, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pReadDel, psQIF)); /* tell the input file object that it must not delete the file on close if the queue * is non-empty - but only if we are not during a simple checkpoint */ - if(bIsCheckpoint != QUEUE_CHECKPOINT && pThis->tVars.disk.pRead != NULL) { - CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); + if(bIsCheckpoint != QUEUE_CHECKPOINT) { + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 0)); } /* we have persisted the queue object. So whenever it comes to an empty queue, @@ -1986,36 +1980,86 @@ finalize_it: /* check if we need to persist the current queue info. If an - * error occurs, thus should be ignored by caller (but we still + * error occurs, this should be ignored by caller (but we still * abide to our regular call interface)... * rgerhards, 2008-01-13 + * nUpdates is the number of updates since the last call to this function. + * It may be > 1 due to batches. -- rgerhards, 2009-05-12 */ -static rsRetVal qqueueChkPersist(qqueue_t *pThis) +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates) { + DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); + assert(nUpdates >= 0); + + if(nUpdates == 0) + FINALIZE; - if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { + pThis->iUpdsSincePersist += nUpdates; + if(pThis->iPersistUpdCnt && pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { qqueuePersist(pThis, QUEUE_CHECKPOINT); pThis->iUpdsSincePersist = 0; } - return RS_RET_OK; +finalize_it: + RETiRet; +} + + +/* persist a queue with all data elements to disk - this is used to handle + * bSaveOnShutdown. We utilize the DA worker to do this. This must only + * be called after all workers have been shut down and if bSaveOnShutdown + * is actually set. Note that this function may potentially run long, + * depending on the queue configuration (e.g. store on remote machine). + * rgerhards, 2009-05-26 + */ +static inline rsRetVal +DoSaveOnShutdown(qqueue_t *pThis) +{ + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + /* we reduce the low water mark, otherwise the DA worker would terminate when + * it is reached. + */ + DBGOPRINT((obj_t*) pThis, "bSaveOnShutdown set, restarting DA worker...\n"); + pThis->bShutdownImmediate = 0; /* would termiante the DA worker! */ + pThis->iLowWtrMrk = 0; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN); /* shutdown worker (only) when done (was _IMMEDIATE!) */ + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* restart DA worker */ + + DBGOPRINT((obj_t*) pThis, "waiting for DA worker to terminate...\n"); + timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); + /* and run the primary queue's DA worker to drain the queue */ + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); + DBGOPRINT((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n", + iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + if(iRetLocal != RS_RET_OK) { + DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " + "continuing, but results are unpredictable\n", iRetLocal); + } + + RETiRet; } /* destructor for the queue object */ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) - pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ - - /* shut down all workers (handles *all* of the persistence logic) - * See function head comment of queueShutdownWorkers () on why we don't call it - * We also do not need to shutdown workers when we are in enqueue-only mode or we are a + /* shut down all workers + * We do not need to shutdown workers when we are in enqueue-only mode or we are a * direct queue - because in both cases we have none... ;) * with a child! -- rgerhards, 2008-01-28 */ if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) - qqueueShutdownWorkers(pThis); + ShutdownWorkers(pThis); + + if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + CHKiRet(DoSaveOnShutdown(pThis)); + } /* finally destruct our (regular) worker thread pool * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, @@ -2051,7 +2095,7 @@ CODESTARTobjDestruct(qqueue) * if need arises (what I doubt...) -- rgerhards, 2008-01-25 */ CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { - dbgoprint((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); + DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); } /* finally, clean up some simple things... */ @@ -2068,15 +2112,13 @@ CODESTARTobjDestruct(qqueue) pthread_cond_destroy(&pThis->belowLightDlyWtrMrk); DESTROY_ATOMIC_HELPER_MUT(pThis->mutQueueSize); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutLogDeq); /* type-specific destructor */ iRet = pThis->qDestruct(pThis); - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); - - if(pThis->pszSpoolDir != NULL) - free(pThis->pszSpoolDir); + free(pThis->pszFilePrefix); + free(pThis->pszSpoolDir); ENDobjDestruct(qqueue) @@ -2090,13 +2132,13 @@ qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) { DEFiRet; - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); + free(pThis->pszFilePrefix); + pThis->pszFilePrefix = NULL; if(pszPrefix == NULL) /* just unset the prefix! */ ABORT_FINALIZE(RS_RET_OK); - if((pThis->pszFilePrefix = malloc(sizeof(uchar) * iLenPrefix + 1)) == NULL) + if((pThis->pszFilePrefix = MALLOC(sizeof(uchar) * iLenPrefix + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFilePrefix, pszPrefix, iLenPrefix + 1); pThis->lenFilePrefix = iLenPrefix; @@ -2126,111 +2168,7 @@ finalize_it: } -/* enqueue a new user data element - * Enqueues the new element and awakes worker thread. - */ -rsRetVal -qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) -{ - DEFiRet; - int iCancelStateSave; - struct timespec t; - - ISOBJ_TYPE_assert(pThis, qqueue); - - /* first check if we need to discard this message (which will cause CHKiRet() to exit) - * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize - * and bRunsDA parameters may not reflect the correct settings here, but they are - * "good enough" in the sense that they can be used to drive the decision. Valgrind's - * threading tools may point this access to be an error, but this is done - * intentional. I do not see this causes problems to us. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); - - /* Please note that this function is not cancel-safe and consequently - * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE - * during its execution. If that is not done, race conditions occur if the - * thread is canceled (most important use case is input module termination). - * rgerhards, 2008-01-08 - */ - if(pThis->qType != QUEUETYPE_DIRECT) { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(pThis->mut); - } - - /* then check if we need to add an assistance disk queue */ - if(pThis->bIsDA) - CHKiRet(qqueueChkStrtDA(pThis)); - - /* handle flow control - * There are two different flow control mechanisms: basic and advanced flow control. - * Basic flow control has always been implemented and protects the queue structures - * in that it makes sure no more data is enqueued than the queue is configured to - * support. Enhanced flow control is being added today. There are some sources which - * can easily be stopped, e.g. a file reader. This is the case because it is unlikely - * that blocking those sources will have negative effects (after all, the file is - * continued to be written). Other sources can somewhat be blocked (e.g. the kernel - * log reader or the local log stream reader): in general, nothing is lost if messages - * from these sources are not picked up immediately. HOWEVER, they can not block for - * an extended period of time, as this either causes message loss or - even worse - some - * other bad effects (e.g. unresponsive system in respect to the main system log socket). - * Finally, there are some (few) sources which can not be blocked at all. UDP syslog is - * a prime example. If a UDP message is not received, it is simply lost. So we can't - * do anything against UDP sockets that come in too fast. The core idea of advanced - * flow control is that we take into account the different natures of the sources and - * select flow control mechanisms that fit these needs. This also means, in the end - * result, that non-blockable sources like UDP syslog receive priority in the system. - * It's a side effect, but a good one ;) -- rgerhards, 2008-03-14 - */ - if(flowCtlType == eFLOWCTL_FULL_DELAY) { - while(pThis->iQueueSize >= pThis->iFullDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); - pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */ - } - } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) { - if(pThis->iQueueSize >= pThis->iLightDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); - timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ - pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */ - } - } - - /* from our regular flow control settings, we are now ready to enqueue the object. - * However, we now need to do a check if the queue permits to add more data. If that - * is not the case, basic flow control enters the field, which means we wait for - * the queue to become ready or drop the new message. -- rgerhards, 2008-03-14 - */ - while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize) - || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0 - && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { - dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); - timeoutComp(&t, pThis->toEnq); - if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { - dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); - objDestruct(pUsr); - ABORT_FINALIZE(RS_RET_QUEUE_FULL); - } - } - - /* and finally enqueue the message */ - CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis); - -finalize_it: - if(pThis->qType != QUEUETYPE_DIRECT) { - /* make sure at least one worker is running. */ - qqueueAdviseMaxWorkers(pThis); - /* and release the mutex */ - d_pthread_mutex_unlock(pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); - dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); - } - - RETiRet; -} - - -/* enqueue a single data object. This currently is a helper to qqueueMultiEnqObj. +/* enqueue a single data object. * Note that the queue mutex MUST already be locked when this function is called. * rgerhards, 2009-06-16 */ @@ -2242,12 +2180,8 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ - CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); - /* then check if we need to add an assistance disk queue */ - if(pThis->bIsDA) - CHKiRet(qqueueChkStrtDA(pThis)); - /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. * Basic flow control has always been implemented and protects the queue structures @@ -2270,12 +2204,12 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) */ if(flowCtlType == eFLOWCTL_FULL_DELAY) { while(pThis->iQueueSize >= pThis->iFullDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */ } } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) { if(pThis->iQueueSize >= pThis->iLightDlyMrk) { - dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */ } @@ -2289,10 +2223,11 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize) || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0 && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { - dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); timeoutComp(&t, pThis->toEnq); +// TODO : handle enqOnly => discard! if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { - dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); + DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); objDestruct(pUsr); ABORT_FINALIZE(RS_RET_QUEUE_FULL); } @@ -2300,12 +2235,12 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis); // TODO: optimize, do in outer function! (but we need parts from v5?) finalize_it: RETiRet; } +/* ------------------------------ multi-enqueue functions ------------------------------ */ /* enqueue multiple user data elements at once. The aim is to provide a faster interface * for object submission. Uses the multi_submit_t helper object. * Please note that this function is not cancel-safe and consequently @@ -2313,9 +2248,12 @@ finalize_it: * during its execution. If that is not done, race conditions occur if the * thread is canceled (most important use case is input module termination). * rgerhards, 2009-06-16 + * Note: there now exists multiple different functions implementing specially + * optimized algorithms for different config cases. -- rgerhards, 2010-06-09 */ -rsRetVal -qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub) +/* now the function for all modes but direct */ +static rsRetVal +qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub) { int iCancelStateSave; int i; @@ -2324,78 +2262,89 @@ qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub) ISOBJ_TYPE_assert(pThis, qqueue); assert(pMultiSub != NULL); - if(pThis->qType != QUEUETYPE_DIRECT) { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(pThis->mut); - } - + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); for(i = 0 ; i < pMultiSub->nElem ; ++i) { -dbgprintf("queueMultiEnq: %d\n", i); CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i])); } + qqueueChkPersist(pThis, pMultiSub->nElem); finalize_it: - if(pThis->qType != QUEUETYPE_DIRECT) { - /* make sure at least one worker is running. */ - qqueueAdviseMaxWorkers(pThis); - /* and release the mutex */ - d_pthread_mutex_unlock(pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); - dbgoprint((obj_t*) pThis, "MultiEnqObj advised worker start\n"); + /* make sure at least one worker is running. */ + qqueueAdviseMaxWorkers(pThis); + /* and release the mutex */ + d_pthread_mutex_unlock(pThis->mut); + pthread_setcancelstate(iCancelStateSave, NULL); + DBGOPRINT((obj_t*) pThis, "MultiEnqObj advised worker start\n"); + + RETiRet; +} + +/* now, the same function, but for direct mode */ +static rsRetVal +qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub) +{ + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pMultiSub != NULL); + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { + CHKiRet(qAddDirect(pThis, (void*)pMultiSub->ppMsgs[i])); } +finalize_it: RETiRet; } +/* ------------------------------ END multi-enqueue functions ------------------------------ */ -/* set queue mode to enqueue only or not - * There is one subtle issue: this method may be called during queue - * construction or while it is running. In the former case, the queue - * mutex does not yet exist (it is NULL), while in the later case it - * must be locked. The function detects the state and operates as - * required. - * rgerhards, 2008-01-16 +/* enqueue a new user data element in direct mode + * NOTE/TODO: This is a TESTER/EXPERIEMENTAL, to be changed to better + * code later on (like multi submit!) 2010-06-10 + * Enqueues the new element and awakes worker thread. */ -static rsRetVal -qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +rsRetVal +qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr) { DEFiRet; - DEFVARS_mutexProtection; + ISOBJ_TYPE_assert(pThis, qqueue); + iRet = qAddDirect(pThis, pUsr); + RETiRet; +} + + +/* enqueue a new user data element + * Enqueues the new element and awakes worker thread. + */ +rsRetVal +qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) +{ + DEFiRet; + int iCancelStateSave; ISOBJ_TYPE_assert(pThis, qqueue); - /* for simplicity, we do one big mutex lock. This method is extremely seldom - * called, so that doesn't matter... -- rgerhards, 2008-01-16 - */ - if(pThis->mut != NULL) { - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); + if(pThis->qType != QUEUETYPE_DIRECT) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); } - if(bEnqOnly == pThis->bEnqOnly) - FINALIZE; /* no change, nothing to do */ - - if(pThis->bQueueStarted) { - /* we need to adjust queue operation only if we are not during initial param setup */ - if(bEnqOnly == 1) { - /* switch to enqueue-only mode */ - /* this means we need to terminate all workers - that's it... */ - dbgoprint((obj_t*) pThis, "switching to enqueue-only mode, terminating all worker threads\n"); - if(pThis->pWtpReg != NULL) - wtpWakeupAllWrkr(pThis->pWtpReg); - if(pThis->pWtpDA != NULL) - wtpWakeupAllWrkr(pThis->pWtpDA); - } else { - /* switch back to regular mode */ - ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); /* we don't need this so far... */ - } - } + CHKiRet(doEnqSingleObj(pThis, flowCtlType, pUsr)); - pThis->bEnqOnly = bEnqOnly; + qqueueChkPersist(pThis, 1); finalize_it: - if(pThis->mut != NULL) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + if(pThis->qType != QUEUETYPE_DIRECT) { + /* make sure at least one worker is running. */ + qqueueAdviseMaxWorkers(pThis); + /* and release the mutex */ + d_pthread_mutex_unlock(pThis->mut); + pthread_setcancelstate(iCancelStateSave, NULL); + DBGOPRINT((obj_t*) pThis, "EnqueueMsg advised worker start\n"); } + RETiRet; } @@ -2419,6 +2368,7 @@ DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int) DEFpropSetMeth(qqueue, bSaveOnShutdown, int) DEFpropSetMeth(qqueue, pUsr, void*) DEFpropSetMeth(qqueue, iDeqSlowdown, int) +DEFpropSetMeth(qqueue, iDeqBatchSize, int) DEFpropSetMeth(qqueue, sizeOnDiskMax, int64) @@ -2437,8 +2387,6 @@ static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp) if(isProp("iQueueSize")) { pThis->iQueueSize = pProp->val.num; - } else if(isProp("iUngottenObjs")) { - pThis->iUngottenObjs = pProp->val.num; } else if(isProp("tVars.disk.sizeOnDisk")) { pThis->tVars.disk.sizeOnDisk = pProp->val.num; } else if(isProp("tVars.disk.bytesRead")) { @@ -2464,6 +2412,8 @@ BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/queue.h b/runtime/queue.h index aafdaa45..1c758134 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -27,8 +27,18 @@ #include <pthread.h> #include "obj.h" #include "wtp.h" +#include "batch.h" #include "stream.h" +/* support for the toDelete list */ +typedef struct toDeleteLst_s toDeleteLst_t; +struct toDeleteLst_s { + qDeqID deqID; + int nElemDeq; /* numbe of elements that were dequeued and as such must now be discarded */ + struct toDeleteLst_s *pNext; +}; + + /* queue types */ typedef enum { QUEUETYPE_FIXED_ARRAY = 0,/* a simple queue made out of a fixed (initially malloced) array fast but memoryhog */ @@ -44,24 +54,15 @@ typedef struct qLinkedList_S { } qLinkedList_t; -typedef struct qWrkThrd_s { - pthread_t thrdID; /* thread ID */ - qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - obj_t *pUsr; /* current user object being processed (or NULL if none) */ - struct queue_s *pQueue; /* my queue (important if only the work thread instance is passed! */ - int iThrd; /* my worker thread array index */ - pthread_cond_t condInitDone; /* signaled when the thread startup is done (once per thread existance) */ - pthread_mutex_t mut; -} qWrkThrd_t; /* type for queue worker threads */ - /* the queue object */ -typedef struct queue_s { +struct queue_s { BEGINobjInstance; queueType_t qType; - bool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ - bool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ - bool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ - bool bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ + int nLogDeq; /* number of elements currently logically dequeued */ + int bShutdownImmediate; /* should all workers cease processing messages? */ + sbool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ + sbool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ + sbool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ int iQueueSize; /* Current number of elements in the queue */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ @@ -72,18 +73,20 @@ typedef struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ - bool bSyncQueueFiles;/* if working with files, sync them after each write? */ + sbool bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ int iFullDlyMrk; /* if the queue is above this mark, FULL_DELAYable message are put on hold */ int iLightDlyMrk; /* if the queue is above this mark, LIGHT_DELAYable message are put on hold */ int iDiscardSeverity;/* messages of this severity above are discarded on too-full queue */ - bool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ + sbool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ int toQShutdown; /* timeout for regular queue shutdown in ms */ int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ + toDeleteLst_t *toDeleteLst;/* this queue's to-delete list */ int toEnq; /* enqueue timeout */ + int iDeqBatchSize; /* max number of elements that shall be dequeued at once */ /* rate limiting settings (will be expanded) */ int iDeqSlowdown; /* slow down dequeue by specified nbr of microseconds */ /* end rate limiting */ @@ -97,19 +100,23 @@ typedef struct queue_s { * applied to detect user configuration errors (and tell me how should we detect what * the user really wanted...). -- rgerhards, 2008-04-02 */ - /* ane dequeue time window */ - rsRetVal (*pConsumer)(void *,void*); /* user-supplied consumer function for dequeued messages */ + /* end dequeue time window */ + rsRetVal (*pConsumer)(void *,batch_t*,int*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the - * user pointer that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 is pointer - * to message) - * rgerhards, 2008-01-28 + * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 + * is pointer to an array of message message pointers), arg3 is a pointer to an interger which is zero + * during normal operations and one if the consumer must urgently shut down. */ /* type-specific handlers (set during construction) */ rsRetVal (*qConstruct)(struct queue_s *pThis); rsRetVal (*qDestruct)(struct queue_s *pThis); rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr); - rsRetVal (*qDel)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDel)(struct queue_s *pThis); /* end type-specific handler */ + /* public entry points (set during construction, permit to set best algorithm for params selected) */ + rsRetVal (*MultiEnq)(qqueue_t *pThis, multi_submit_t *pMultiSub); + /* end public entry points */ /* synchronization variables */ pthread_mutex_t mutThrdMgmt; /* mutex for the queue's thread management */ pthread_mutex_t *mut; /* mutex for enqueing and dequeueing messages */ @@ -117,7 +124,6 @@ typedef struct queue_s { pthread_cond_t belowFullDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t belowLightDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t condDAReady;/* signalled when the DA queue is fully initialized and ready for processing */ - int bChildIsDone; /* set to 1 when the child DA queue has finished processing, 0 otherwise */ int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* the following variables are always present, because they @@ -132,43 +138,35 @@ typedef struct queue_s { int iNumberFiles; /* how many files make up the queue? */ int64 iMaxFileSize; /* max size for a single queue file */ int64 sizeOnDiskMax; /* maximum size on disk allowed */ + qDeqID deqIDAdd; /* next dequeue ID to use during add to queue store */ + qDeqID deqIDDel; /* queue store delete position */ int bIsDA; /* is this queue disk assisted? */ - int bRunsDA; /* is this queue actually *running* disk assisted? */ struct queue_s *pqDA; /* queue for disk-assisted modes */ struct queue_s *pqParent;/* pointer to the parent (if this is a child queue) */ int bDAEnqOnly; /* EnqOnly setting for DA queue */ - /* some data elements for the queueUngetObj() functionality. This list should always be short - * and is always kept in memory - */ - qLinkedList_t *pUngetRoot; - qLinkedList_t *pUngetLast; - int iUngottenObjs; /* number of objects currently in the "ungotten" list */ /* now follow queueing mode specific data elements */ union { /* different data elements based on queue type (qType) */ struct { - long head, tail; + long deqhead, head, tail; void** pBuf; /* the queued user data structure */ } farray; struct { - qLinkedList_t *pRoot; + qLinkedList_t *pDeqRoot; + qLinkedList_t *pDelRoot; qLinkedList_t *pLast; } linklist; struct { int64 sizeOnDisk; /* current amount of disk space used */ int64 bytesRead; /* number of bytes read from current (undeleted!) file */ - strm_t *pWrite; /* current file to be written */ - strm_t *pRead; /* current file to be read */ + strm_t *pWrite; /* current file to be written */ + strm_t *pReadDeq; /* current file for dequeueing */ + strm_t *pReadDel; /* current file for deleting */ } disk; } tVars; DEF_ATOMIC_HELPER_MUT(mutQueueSize); -} qqueue_t; - -/* some symbolic constants for easier reference */ -#define QUEUE_MODE_ENQDEQ 0 -#define QUEUE_MODE_ENQONLY 1 + DEF_ATOMIC_HELPER_MUT(mutLogDeq); +}; -#define QUEUE_IDX_DA_WORKER 0 /* index for the DA worker (fixed) */ -#define QUEUE_PTR_DA_WORKER(x) (&((pThis)->pWrkThrds[0])) /* the define below is an "eternal" timeout for the timeout settings which require a value. * It is one day, which is not really eternal, but comes close to it if we think about @@ -179,13 +177,14 @@ typedef struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); -rsRetVal qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub); +rsRetVal qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr); rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*)); +rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); PROTOTYPEpropSetMeth(qqueue, bSyncQueueFiles, int); @@ -204,6 +203,7 @@ PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int); PROTOTYPEpropSetMeth(qqueue, pUsr, void*); PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int); PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); +PROTOTYPEpropSetMeth(qqueue, iDeqBatchSize, int); #define qqueueGetID(pThis) ((unsigned long) pThis) #endif /* #ifndef QUEUE_H_INCLUDED */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index c209ae30..a9794840 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -80,6 +80,8 @@ #include "prop.h" #include "rule.h" #include "ruleset.h" +#include "parser.h" +#include "strgen.h" #include "atomic.h" /* forward definitions */ @@ -152,8 +154,6 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; CHKiRet(glblClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "datetime"; - CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ctok_token"; @@ -184,6 +184,10 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "parser"; + CHKiRet(parserClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "strgen"; + CHKiRet(strgenClassInit(NULL)); /* dummy "classes" */ if(ppErrObj != NULL) *ppErrObj = "str"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 9dd45b38..34eaedca 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -27,6 +27,12 @@ #define INCLUDED_RSYSLOG_H /* ############################################################# * + * # Some constant values # * + * ############################################################# */ +#define CONST_LEN_TIMESTAMP_3164 15 /* number of chars (excluding \0!) in a RFC3164 timestamp */ +#define CONST_LEN_TIMESTAMP_3339 32 /* number of chars (excluding \0!) in a RFC3339 timestamp */ + +/* ############################################################# * * # Config Settings # * * ############################################################# */ #define RS_STRINGBUF_ALLOC_INCREMENT 128 @@ -40,7 +46,27 @@ #define CONF_TAG_BUFSIZE 32 #define CONF_HOSTNAME_BUFSIZE 32 #define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ - +#define CONF_MIN_SIZE_FOR_COMPRESS 60 /* config param: minimum message size to try compression. The smaller + * the message, the less likely is any compression gain. We check for + * gain before we submit the message. But to do so we still need to + * do the (costly) compress() call. The following setting sets a size + * for which no call to compress() is done at all. This may result in + * a few more bytes being transmited but better overall performance. + * Note: I have not yet checked the minimum UDP packet size. It might be + * that we do not save anything by compressing very small messages, because + * UDP might need to pad ;) + * rgerhards, 2006-11-30 + */ + +#define CONF_OMOD_NUMSTRINGS_MAXSIZE 2 /* cache for pointers to output module buffer pointers. All + * rsyslog-provided plugins do NOT need more than two buffers. If + * more are needed (future developments, third-parties), rsyslog + * must be recompiled with a larger parameter. Hardcoding this + * saves us some overhead, both in runtime in code complexity. As + * it is doubtful if ever more than 2 parameters are needed, the + * approach taken here is considered appropriate. + * rgerhards, 2010-06-24 + */ /* ############################################################# * * # End Config Settings # * @@ -60,9 +86,29 @@ #endif +/* the rsyslog core provides information about present feature to plugins + * asking it. Below are feature-test macros which must be used to query + * features. Note that this must be powers of two, so that multiple queries + * can be combined. -- rgerhards, 2009-04-27 + */ +#define CORE_FEATURE_BATCHING 1 +/*#define CORE_FEATURE_whatever 2 ... and so on ... */ + +/* under Solaris (actually only SPARC), we need to redefine some types + * to be void, so that we get void* pointers. Otherwise, we will see + * alignment errors. + */ +/* some universal fixed size integer defines ... */ +typedef long long int64; +typedef long long unsigned uint64; +typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef char intTiny; /* 0..127! */ +typedef unsigned char uintTiny; /* 0..255! */ + /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ +typedef struct aUsrp_s aUsrp_t; typedef struct thrdInfo thrdInfo_t; typedef struct obj_s obj_t; typedef struct ruleset_s ruleset_t; @@ -72,6 +118,7 @@ typedef struct NetAddr netAddr_t; typedef struct netstrms_s netstrms_t; typedef struct netstrm_s netstrm_t; typedef struct nssel_s nssel_t; +typedef struct nspoll_s nspoll_t; typedef enum nsdsel_waitOp_e nsdsel_waitOp_t; typedef struct nsd_ptcp_s nsd_ptcp_t; typedef struct nsd_gtls_s nsd_gtls_t; @@ -79,7 +126,10 @@ typedef struct nsd_gsspi_s nsd_gsspi_t; typedef struct nsd_nss_s nsd_nss_t; typedef struct nsdsel_ptcp_s nsdsel_ptcp_t; typedef struct nsdsel_gtls_s nsdsel_gtls_t; +typedef struct nsdpoll_ptcp_s nsdpoll_ptcp_t; +typedef struct wti_s wti_t; typedef struct msg msg_t; +typedef struct queue_s qqueue_t; typedef struct prop_s prop_t; typedef struct interface_s interface_t; typedef struct objInfo_s objInfo_t; @@ -92,7 +142,16 @@ typedef struct tcps_sess_s tcps_sess_t; typedef struct strmsrv_s strmsrv_t; typedef struct strms_sess_s strms_sess_t; typedef struct vmstk_s vmstk_t; +typedef struct batch_obj_s batch_obj_t; +typedef struct batch_s batch_t; +typedef struct wtp_s wtp_t; +typedef struct modInfo_s modInfo_t; +typedef struct parser_s parser_t; +typedef struct parserList_s parserList_t; +typedef struct strgen_s strgen_t; +typedef struct strgenList_s strgenList_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ +typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? @@ -105,26 +164,23 @@ typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? typedef void * obj_t_ptr; typedef void nsd_t; typedef void nsdsel_t; + typedef void nsdpoll_t; #else typedef obj_t *obj_t_ptr; typedef obj_t nsd_t; typedef obj_t nsdsel_t; + typedef obj_t nsdpoll_t; #endif -/* some universal 64 bit define... */ -typedef long long int64; -typedef long long unsigned uint64; -typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ -typedef char intTiny; /* 0..127! */ -typedef uchar uintTiny; /* 0..255! */ - #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif -typedef char bool; /* I intentionally use char, to keep it slim so that many fit into the CPU cache! */ +typedef struct epoll_event epoll_event_t; + +typedef char sbool; /* (small bool) I intentionally use char, to keep it slim so that many fit into the CPU cache! */ /* settings for flow control * TODO: is there a better place for them? -- rgerhards, 2008-03-14 @@ -364,7 +420,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ - RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */ + RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended */ RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */ RS_RET_NON_SIZELIMITCMD = -2125, /**< size limit for file defined, but no size limit command given */ RS_RET_SIZELIMITCMD_DIDNT_RESOLVE = -2126, /**< size limit command did not resolve situation */ @@ -377,6 +433,22 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_OPEN_KLOG = -2145, /**< error opening the kernel log socket (primarily solaris) */ RS_RET_ERR_AQ_CONLOG = -2146, /**< error aquiring console log (on solaris) */ RS_RET_ERR_DOOR = -2147, /**< some problems with handling the Solaris door functionality */ + RS_RET_NO_SRCNAME_TPL = -2150, /**< sourcename template was not specified where one was needed (omudpspoof spoof addr) */ + RS_RET_HOST_NOT_SPECIFIED = -2151, /**< (target) host was not specified where it was needed */ + RS_RET_ERR_LIBNET_INIT = -2152, /**< error initializing libnet */ + RS_RET_FORCE_TERM = -2153, /**< thread was forced to terminate by bShallShutdown, a state, not an error */ + RS_RET_RULES_QUEUE_EXISTS = -2154,/**< we were instructed to create a new ruleset queue, but one already exists */ + RS_RET_NO_CURR_RULESET = -2155,/**< no current ruleset exists (but one is required) */ + RS_RET_NO_MSG_PASSING = -2156,/**< output module interface parameter passing mode "MSG" is not available but required */ + RS_RET_RULESET_NOT_FOUND = -2157,/**< a required ruleset could not be found */ + RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */ + RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */ + RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */ + RS_RET_EINTR = -2161, /**< EINTR occured during a system call (not necessarily an error) */ + RS_RET_ERR_EPOLL = -2162, /**< epoll() returned with an unexpected error code */ + RS_RET_ERR_EPOLL_CTL = -2163, /**< epol_ctll() returned with an unexpected error code */ + RS_RET_TIMEOUT = -2164, /**< timeout occured during operation */ + RS_RET_RCV_ERR = -2165, /**< error occured during socket rcv operation */ RS_RET_NO_SOCK_CONFIGURED = -2166, /**< no socket (name) was configured where one is required */ RS_RET_NO_LSTN_DEFINED = -2172, /**< no listener defined (e.g. inside an input module) */ RS_RET_EPOLL_CR_FAILED = -2173, /**< epoll_create() failed */ @@ -387,10 +459,12 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ /* some generic error/status codes */ + RS_RET_OK = 0, /**< operation successful */ RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */ RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */ RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */ - RS_RET_OK = 0 /**< operation successful */ + RS_RET_IDLE = 4, /**< operation successful, but callee is idle (e.g. because queue is empty) */ + RS_RET_TERMINATE_WHEN_IDLE = 5 /**< operation successful, function is requested to terminate when idle */ }; /* some helpful macros to work with srRetVals. diff --git a/runtime/rule.c b/runtime/rule.c index 4c2c9edb..42773768 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -39,8 +39,8 @@ #include "vm.h" #include "var.h" #include "srUtils.h" +#include "batch.h" #include "unicode-helper.h" -#include "dirty.h" /* for getFIOPName */ /* static data */ DEFobjStaticHelpers @@ -49,6 +49,35 @@ DEFobjCurrIf(expr) DEFobjCurrIf(var) DEFobjCurrIf(vm) + +/* support for simple textual representation of FIOP names + * rgerhards, 2005-09-27 + */ +static char* +getFIOPName(unsigned iFIOP) +{ + char *pRet; + switch(iFIOP) { + case FIOP_CONTAINS: + pRet = "contains"; + break; + case FIOP_ISEQUAL: + pRet = "isequal"; + break; + case FIOP_STARTSWITH: + pRet = "startswith"; + break; + case FIOP_REGEX: + pRet = "regex"; + break; + default: + pRet = "NOP"; + break; + } + return pRet; +} + + /* iterate over all actions, this is often needed, for example when HUP processing * must be done or a shutdown is pending. */ @@ -59,40 +88,20 @@ iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) } - /* helper to processMsg(), used to call the configured actions. It is * executed from within llExecFunc() of the action list. * rgerhards, 2007-08-02 */ -typedef struct processMsgDoActions_s { - int bPrevWasSuspended; /* was the previous action suspended? */ - msg_t *pMsg; -} processMsgDoActions_t; -DEFFUNC_llExecFunc(processMsgDoActions) +DEFFUNC_llExecFunc(processBatchDoActions) { DEFiRet; rsRetVal iRetMod; /* return value of module - we do not always pass that back */ action_t *pAction = (action_t*) pData; - processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; - - assert(pAction != NULL); - - if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - dbgprintf("not calling action because the previous one is not suspended\n"); - ABORT_FINALIZE(RS_RET_OK); - } + batch_t *pBatch = (batch_t*) pParam; - iRetMod = actionCallAction(pAction, pDoActData->pMsg); - if(iRetMod == RS_RET_DISCARDMSG) { - ABORT_FINALIZE(RS_RET_DISCARDMSG); - } else if(iRetMod == RS_RET_SUSPENDED) { - /* indicate suspension for next module to be called */ - pDoActData->bPrevWasSuspended = 1; - } else { - pDoActData->bPrevWasSuspended = 0; - } + DBGPRINTF("Processing next action\n"); + iRetMod = pAction->submitToActQ(pAction, pBatch); -finalize_it: RETiRet; } @@ -101,7 +110,7 @@ finalize_it: * provided filter condition. */ static rsRetVal -shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) +shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, sbool *bProcessMsg) { DEFiRet; unsigned short pbMustBeFreed; @@ -250,26 +259,29 @@ finalize_it: -/* Process (consume) a received message. Calls the actions configured. +/* Process (consume) a batch of messages. Calls the actions configured. * rgerhards, 2005-10-13 */ static rsRetVal -processMsg(rule_t *pThis, msg_t *pMsg) +processBatch(rule_t *pThis, batch_t *pBatch) { - int bProcessMsg; - processMsgDoActions_t DoActData; + int i; DEFiRet; ISOBJ_TYPE_assert(pThis, rule); - assert(pMsg != NULL); - - /* first check the filters... */ - CHKiRet(shouldProcessThisMessage(pThis, pMsg, &bProcessMsg)); - if(bProcessMsg) { - DoActData.pMsg = pMsg; - DoActData.bPrevWasSuspended = 0; - CHKiRet(llExecFunc(&pThis->llActList, processMsgDoActions, (void*)&DoActData)); + assert(pBatch != NULL); + + /* first check the filters and reset status variables */ + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + CHKiRet(shouldProcessThisMessage(pThis, (msg_t*)(pBatch->pElem[i].pUsrp), + &(pBatch->pElem[i].bFilterOK))); + // TODO: really abort on error? 2010-06-10 + if(pBatch->pElem[i].bFilterOK) { + /* re-init only when actually needed (cache write cost!) */ + pBatch->pElem[i].bPrevWasSuspended = 0; + } } + CHKiRet(llExecFunc(&pThis->llActList, processBatchDoActions, pBatch)); finalize_it: RETiRet; @@ -412,7 +424,7 @@ CODESTARTobjQueryInterface(rule) pIf->DebugPrint = ruleDebugPrint; pIf->IterateAllActions = iterateAllActions; - pIf->ProcessMsg = processMsg; + pIf->ProcessBatch = processBatch; pIf->SetAssRuleset = setAssRuleset; pIf->GetAssRuleset = getAssRuleset; finalize_it: diff --git a/runtime/rule.h b/runtime/rule.h index 99ac44e7..309a2ed8 100644 --- a/runtime/rule.h +++ b/runtime/rule.h @@ -47,7 +47,7 @@ struct rule_s { fiop_t operation; regex_t *regex_cache; /* cache for compiled REs, if such are used */ cstr_t *pCSCompValue; /* value to "compare" against */ - bool isNegated; + sbool isNegated; propid_t propID; /* ID of the requested property */ } prop; expr_t *f_expr; /* expression object */ @@ -64,11 +64,12 @@ BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis); rsRetVal (*Destruct)(rule_t **ppThis); rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); - rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg); + rsRetVal (*ProcessBatch)(rule_t *pThis, batch_t *pBatch); rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*); ruleset_t* (*GetAssRuleset)(rule_t *pThis); ENDinterface(rule) -#define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define ruleCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* change for v2: ProcessMsg replaced by ProcessBatch - 2010-06-10 */ /* prototypes */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c index af61f24f..0584e8d6 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -40,22 +40,28 @@ #include "rsyslog.h" #include "obj.h" +#include "cfsysline.h" #include "msg.h" #include "ruleset.h" #include "rule.h" #include "errmsg.h" +#include "parser.h" +#include "batch.h" #include "unicode-helper.h" - -static rsRetVal debugPrintAll(void); // TODO: remove! +#include "dirty.h" /* for main ruleset queue creation */ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(rule) +DEFobjCurrIf(parser) linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ ruleset_t *pCurrRuleset = NULL; /* currently "active" ruleset */ -ruleset_t *pDfltRuleset = NULL; /* currentl default ruleset, e.g. for binding to actions which have no other */ +ruleset_t *pDfltRuleset = NULL; /* current default ruleset, e.g. for binding to actions which have no other */ + +/* forward definitions */ +static rsRetVal processBatch(batch_t *pBatch); /* ---------- linked-list key handling functions ---------- */ @@ -132,39 +138,114 @@ finalize_it: -/* helper to processMsg(), used to call the configured actions. It is +/* helper to processBatch(), used to call the configured actions. It is * executed from within llExecFunc() of the action list. * rgerhards, 2007-08-02 */ -DEFFUNC_llExecFunc(processMsgDoRules) +DEFFUNC_llExecFunc(processBatchDoRules) { + rsRetVal iRet; ISOBJ_TYPE_assert(pData, rule); - return rule.ProcessMsg((rule_t*) pData, (msg_t*) pParam); + dbgprintf("Processing next rule\n"); + iRet = rule.ProcessBatch((rule_t*) pData, (batch_t*) pParam); +dbgprintf("ruleset: get iRet %d from rule.ProcessMsg()\n", iRet); + return iRet; } -/* Process (consume) a received message. Calls the actions configured. + +/* This function is similar to processBatch(), but works on a batch that + * contains rules from multiple rulesets. In this case, we can not push + * the whole batch through the ruleset. Instead, we examine it and + * partition it into sub-rulesets which we then push through the system. + * Note that when we evaluate which message must be processed, we do NOT need + * to look at bFilterOK, because this value is only set in a later processing + * stage. Doing so caused a bug during development ;) + * rgerhards, 2010-06-15 + */ +static inline rsRetVal +processBatchMultiRuleset(batch_t *pBatch) +{ + ruleset_t *currRuleset; + batch_t snglRuleBatch; + int i; + int iStart; /* start index of partial batch */ + int iNew; /* index for new (temporary) batch */ + DEFiRet; + + CHKiRet(batchInit(&snglRuleBatch, pBatch->nElem)); + snglRuleBatch.pbShutdownImmediate = pBatch->pbShutdownImmediate; + + while(1) { /* loop broken inside */ + /* search for first unprocessed element */ + for(iStart = 0 ; iStart < pBatch->nElem && pBatch->pElem[iStart].state == BATCH_STATE_DISC ; ++iStart) + /* just search, no action */; + + if(iStart == pBatch->nElem) + FINALIZE; /* everything processed */ + + /* prepare temporary batch */ + currRuleset = batchElemGetRuleset(pBatch, iStart); + iNew = 0; + for(i = iStart ; i < pBatch->nElem ; ++i) { + if(batchElemGetRuleset(pBatch, i) == currRuleset) { + batchCopyElem(&(snglRuleBatch.pElem[iNew++]), &(pBatch->pElem[i])); + /* We indicate the element also as done, so it will not be processed again */ + pBatch->pElem[i].state = BATCH_STATE_DISC; + } + } + snglRuleBatch.nElem = iNew; /* was left just right by the for loop */ + batchSetSingleRuleset(&snglRuleBatch, 1); + /* process temp batch */ + processBatch(&snglRuleBatch); + } + batchFree(&snglRuleBatch); + +finalize_it: + RETiRet; +} + +/* Process (consume) a batch of messages. Calls the actions configured. + * If the whole batch uses a singel ruleset, we can process the batch as + * a whole. Otherwise, we need to process it slower, on a message-by-message + * basis (what can be optimized to a per-ruleset basis) * rgerhards, 2005-10-13 */ static rsRetVal -processMsg(msg_t *pMsg) +processBatch(batch_t *pBatch) { ruleset_t *pThis; DEFiRet; - assert(pMsg != NULL); - - pThis = (pMsg->pRuleset == NULL) ? pDfltRuleset : pMsg->pRuleset; - ISOBJ_TYPE_assert(pThis, ruleset); - - CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); + assert(pBatch != NULL); + +dbgprintf("ZZZ: processBatch: batch of %d elements must be processed\n", pBatch->nElem); + if(pBatch->bSingleRuleset) { + pThis = batchGetRuleset(pBatch); + if(pThis == NULL) + pThis = pDfltRuleset; + ISOBJ_TYPE_assert(pThis, ruleset); + CHKiRet(llExecFunc(&pThis->llRules, processBatchDoRules, pBatch)); + } else { + CHKiRet(processBatchMultiRuleset(pBatch)); + } finalize_it: - if(iRet == RS_RET_DISCARDMSG) - iRet = RS_RET_OK; - +dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet); RETiRet; } + +/* return the ruleset-assigned parser list. NULL means use the default + * parser list. + * rgerhards, 2009-11-04 + */ +static parserList_t* +GetParserList(msg_t *pMsg) +{ + return (pMsg->pRuleset == NULL) ? pDfltRuleset->pParserLst : pMsg->pRuleset->pParserLst; +} + + /* Add a new rule to the end of the current rule set. We do a number * of checks and ignore the rule if it does not pass them. */ @@ -214,6 +295,19 @@ GetCurrent(void) } +/* get main queue associated with ruleset. If no ruleset-specifc main queue + * is set, the primary main message queue is returned. + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static qqueue_t* +GetRulesetQueue(ruleset_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, ruleset); + return (pThis->pQueue == NULL) ? pMsgQueue : pThis->pQueue; +} + + /* Find the ruleset with the given name and return a pointer to its object. */ static rsRetVal @@ -319,6 +413,12 @@ finalize_it: BEGINobjDestruct(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(ruleset) dbgprintf("destructing ruleset %p, name %p\n", pThis, pThis->pszName); + if(pThis->pQueue != NULL) { + qqueueDestruct(&pThis->pQueue); + } + if(pThis->pParserLst != NULL) { + parser.DestructParserList(&pThis->pParserLst); + } llDestroy(&pThis->llRules); free(pThis->pszName); ENDobjDestruct(ruleset) @@ -385,6 +485,81 @@ debugPrintAll(void) } +/* Create a ruleset-specific "main" queue for this ruleset. If one is already + * defined, an error message is emitted but nothing else is done. + * Note: we use the main message queue parameters for queue creation and access + * syslogd.c directly to obtain these. This is far from being perfect, but + * considered acceptable for the time being. + * rgerhards, 2009-10-27 + */ +static rsRetVal +rulesetCreateQueue(void __attribute__((unused)) *pVal, int *pNewVal) +{ + DEFiRet; + + if(pCurrRuleset == NULL) { + errmsg.LogError(0, RS_RET_NO_CURR_RULESET, "error: currently no specific ruleset specified, thus a " + "queue can not be added to it"); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } + + if(pCurrRuleset->pQueue != NULL) { + errmsg.LogError(0, RS_RET_RULES_QUEUE_EXISTS, "error: ruleset already has a main queue, can not " + "add another one"); + ABORT_FINALIZE(RS_RET_RULES_QUEUE_EXISTS); + } + + if(pNewVal == 0) + FINALIZE; /* if it is turned off, we do not need to change anything ;) */ + + dbgprintf("adding a ruleset-specific \"main\" queue"); + CHKiRet(createMainQueue(&pCurrRuleset->pQueue, UCHAR_CONSTANT("ruleset"))); + +finalize_it: + RETiRet; +} + + +/* Add a ruleset specific parser to the ruleset. Note that adding the first + * parser automatically disables the default parsers. If they are needed as well, + * the must be added via explicit config directives. + * Note: this is the only spot in the code that requires the parser object. In order + * to solve some class init bootstrap sequence problems, we get the object handle here + * instead of during module initialization. Note that objUse() is capable of being + * called multiple times. + * rgerhards, 2009-11-04 + */ +static rsRetVal +rulesetAddParser(void __attribute__((unused)) *pVal, uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + assert(pCurrRuleset != NULL); + + CHKiRet(objUse(parser, CORE_COMPONENT)); + iRet = parser.FindParser(&pParser, pName); + if(iRet == RS_RET_PARSER_NOT_FOUND) { + errmsg.LogError(0, RS_RET_PARSER_NOT_FOUND, "error: parser '%s' unknown at this time " + "(maybe defined too late in rsyslog.conf?)", pName); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } else if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "error trying to find parser '%s'\n", pName); + FINALIZE; + } + + CHKiRet(parser.AddParserToList(&pCurrRuleset->pParserLst, pParser)); + + dbgprintf("added parser '%s' to ruleset '%s'\n", pName, pCurrRuleset->pszName); +RUNLOG_VAR("%p", pCurrRuleset->pParserLst); + +finalize_it: + d_free(pName); /* no longer needed */ + + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-02-21 */ @@ -407,13 +582,15 @@ CODESTARTobjQueryInterface(ruleset) pIf->IterateAllActions = iterateAllActions; pIf->DestructAllActions = destructAllActions; pIf->AddRule = addRule; - pIf->ProcessMsg = processMsg; + pIf->ProcessBatch = processBatch; pIf->SetName = setName; pIf->DebugPrintAll = debugPrintAll; pIf->GetCurrent = GetCurrent; pIf->GetRuleset = GetRuleset; pIf->SetDefaultRuleset = SetDefaultRuleset; pIf->SetCurrRuleset = SetCurrRuleset; + pIf->GetRulesetQueue = GetRulesetQueue; + pIf->GetParserList = GetParserList; finalize_it: ENDobjQueryInterface(ruleset) @@ -425,6 +602,7 @@ BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ llDestroy(&llRulesets); objRelease(errmsg, CORE_COMPONENT); objRelease(rule, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); ENDObjClassExit(ruleset) @@ -443,6 +621,10 @@ BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* prepare global data */ CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + + /* config file handlers */ + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetparser", 0, eCmdHdlrGetWord, rulesetAddParser, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetcreatemainqueue", 0, eCmdHdlrBinary, rulesetCreateQueue, NULL, NULL)); ENDObjClassInit(ruleset) /* vi:set ai: diff --git a/runtime/ruleset.h b/runtime/ruleset.h index 32571687..acebd17a 100644 --- a/runtime/ruleset.h +++ b/runtime/ruleset.h @@ -25,6 +25,7 @@ #ifndef INCLUDED_RULESET_H #define INCLUDED_RULESET_H +#include "queue.h" #include "linkedlist.h" /* the ruleset object */ @@ -32,6 +33,8 @@ struct ruleset_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ uchar *pszName; /* name of our ruleset */ + qqueue_t *pQueue; /* "main" message queue, if the ruleset has its own (else NULL) */ + parserList_t *pParserLst;/* list of parsers to use for this ruleset */ }; /* interfaces */ @@ -45,13 +48,16 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ rsRetVal (*DestructAllActions)(void); rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); - rsRetVal (*ProcessMsg)(msg_t *pMsg); + rsRetVal (*ProcessBatch)(batch_t*); rsRetVal (*GetRuleset)(ruleset_t **ppThis, uchar*); rsRetVal (*SetDefaultRuleset)(uchar*); rsRetVal (*SetCurrRuleset)(uchar*); ruleset_t* (*GetCurrent)(void); + qqueue_t* (*GetRulesetQueue)(ruleset_t*); + /* v3, 2009-11-04 */ + parserList_t* (*GetParserList)(msg_t *); ENDinterface(ruleset) -#define rulesetCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define rulesetCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/srUtils.h b/runtime/srUtils.h index 16766312..c4f73e16 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -110,30 +110,17 @@ rsRetVal getFileSize(uchar *pszName, off_t *pSize); /* some useful constants */ #define DEFVARS_mutexProtection\ - int iCancelStateSave; \ int bLockedOpIsLocked=0 #define BEGIN_MTX_PROTECTED_OPERATIONS(mut, bMustLock) \ if(bMustLock == LOCK_MUTEX) { \ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ d_pthread_mutex_lock(mut); \ + assert(bLockedOpIsLocked == 0); \ bLockedOpIsLocked = 1; \ } #define END_MTX_PROTECTED_OPERATIONS(mut) \ if(bLockedOpIsLocked) { \ d_pthread_mutex_unlock(mut); \ - pthread_setcancelstate(iCancelStateSave, NULL); \ + bLockedOpIsLocked = 0; \ } -/* The unconditional versions of the macro always lock the mutex. They are preferred in - * complex scenarios, where the simple ones might get mixed up by multiple calls. - */ -#define DEFVARS_mutexProtection_uncond\ - int iCancelStateSave -#define BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ - d_pthread_mutex_lock(mut); -#define END_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ - d_pthread_mutex_unlock(mut); \ - pthread_setcancelstate(iCancelStateSave, NULL); - #endif diff --git a/runtime/srutils.c b/runtime/srutils.c index 1452c9b7..d357cd77 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -46,6 +46,9 @@ #include "srUtils.h" #include "obj.h" +#if _POSIX_TIMERS <= 0 +#include <sys/time.h> +#endif /* here we host some syslog specific names. There currently is no better place * to do it, but over here is also not ideal... -- rgerhards, 2008-02-14 @@ -158,7 +161,7 @@ uchar *srUtilStrDup(uchar *pOld, size_t len) assert(pOld != NULL); - if((pNew = malloc(len + 1)) != NULL) + if((pNew = MALLOC(len + 1)) != NULL) memcpy(pNew, pOld, len + 1); return pNew; @@ -197,7 +200,7 @@ int makeFileParentDirs(uchar *szFile, size_t lenFile, mode_t mode, assert(lenFile > 0); len = lenFile + 1; /* add one for '\0'-byte */ - if((pszWork = malloc(sizeof(uchar) * len)) == NULL) + if((pszWork = MALLOC(sizeof(uchar) * len)) == NULL) return -1; memcpy(pszWork, szFile, len); for(p = pszWork+1 ; *p ; p++) @@ -346,7 +349,7 @@ rsRetVal genFileName(uchar **ppName, uchar *pDirName, size_t lenDirName, uchar * } lenName = lenDirName + 1 + lenFName + lenBuf + 1; /* last +1 for \0 char! */ - if((pName = malloc(sizeof(uchar) * lenName)) == NULL) + if((pName = MALLOC(sizeof(uchar) * lenName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); /* got memory, now construct string */ @@ -392,10 +395,22 @@ int getNumberDigits(long lNum) rsRetVal timeoutComp(struct timespec *pt, long iTimeout) { +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ + +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, pt); +# else + gettimeofday(&tv, NULL); + pt->tv_sec = tv.tv_sec; + pt->tv_nsec = tv.tv_usec * 1000; +# endif pt->tv_sec += iTimeout / 1000; pt->tv_nsec += (iTimeout % 1000) * 1000000; /* think INTEGER arithmetic! */ if(pt->tv_nsec > 999999999) { /* overrun? */ @@ -417,11 +432,21 @@ timeoutVal(struct timespec *pt) { struct timespec t; long iTimeout; - BEGINfunc +# if _POSIX_TIMERS <= 0 + struct timeval tv; +# endif + BEGINfunc assert(pt != NULL); /* compute timeout */ +# if _POSIX_TIMERS > 0 + /* this is the "regular" code */ clock_gettime(CLOCK_REALTIME, &t); +# else + gettimeofday(&tv, NULL); + t.tv_sec = tv.tv_sec; + t.tv_nsec = tv.tv_usec * 1000; +# endif iTimeout = (pt->tv_nsec - t.tv_nsec) / 1000000; iTimeout += (pt->tv_sec - t.tv_sec) * 1000; diff --git a/runtime/stream.c b/runtime/stream.c index e6680adc..b4295762 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -261,7 +261,7 @@ static rsRetVal strmOpenFile(strm_t *pThis) pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); } else { if(pThis->pszDir == NULL) { - if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) + if((pThis->pszCurrFName = ustrdup(pThis->pszFName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } else { CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, @@ -626,7 +626,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) * to make sure we can write out everything with a SINGLE api call! * We add another 128 bytes to take care of the gzip header and "all eventualities". */ - CHKmalloc(pThis->pZipBuf = (Bytef*) malloc(sizeof(uchar) * (pThis->sIOBufSize + 128))); + CHKmalloc(pThis->pZipBuf = (Bytef*) MALLOC(sizeof(uchar) * (pThis->sIOBufSize + 128))); } } @@ -658,7 +658,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) pthread_cond_init(&pThis->isEmpty, 0); pThis->iCnt = pThis->iEnq = pThis->iDeq = 0; for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) { - CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); } pThis->pIOBuf = pThis->asyncBuf[0].pBuf; pThis->bStopWriter = 0; @@ -666,7 +666,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis); } else { /* we work synchronously, so we need to alloc a fixed pIOBuf */ - CHKmalloc(pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); } finalize_it: @@ -924,7 +924,7 @@ asyncWriterThread(void *pPtr) { int iDeq; struct timespec t; - bool bTimedOut = 0; + sbool bTimedOut = 0; strm_t *pThis = (strm_t*) pPtr; ISOBJ_TYPE_assert(pThis, strm); @@ -937,7 +937,6 @@ asyncWriterThread(void *pPtr) while(1) { /* loop broken inside */ d_pthread_mutex_lock(&pThis->mut); -dbgprintf("XXX: asyncWriterThread iterating %s\n", pThis->pszFName); while(pThis->iCnt == 0) { if(pThis->bStopWriter) { pthread_cond_broadcast(&pThis->isEmpty); @@ -953,7 +952,6 @@ dbgprintf("XXX: asyncWriterThread iterating %s\n", pThis->pszFName); bTimedOut = 0; timeoutComp(&t, pThis->iFlushInterval * 2000); /* *1000 millisconds */ // TODO: check the 2000?!? if(pThis->bDoTimedWait) { -dbgprintf("asyncWriter thread going to timeout sleep\n"); if(pthread_cond_timedwait(&pThis->notEmpty, &pThis->mut, &t) != 0) { int err = errno; if(err == ETIMEDOUT) { @@ -967,16 +965,13 @@ dbgprintf("asyncWriter thread going to timeout sleep\n"); } } } else { -dbgprintf("asyncWriter thread going to eternal sleep\n"); d_pthread_cond_wait(&pThis->notEmpty, &pThis->mut); } -dbgprintf("asyncWriter woke up\n"); } bTimedOut = 0; /* we may have timed out, but there *is* work to do... */ iDeq = pThis->iDeq++ % STREAM_ASYNC_NUMBUFS; -dbgprintf("asyncWriter writes data\n"); doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf); // TODO: error check????? 2009-07-06 @@ -1091,7 +1086,7 @@ doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { z_stream zstrm; int zRet; /* zlib return state */ - bool bzInitDone = FALSE; + sbool bzInitDone = FALSE; DEFiRet; assert(pThis != NULL); assert(pBuf != NULL); @@ -1385,7 +1380,7 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) if(pThis->pszFName != NULL) free(pThis->pszFName); - if((pThis->pszFName = malloc(sizeof(uchar) * (iLenName + 1))) == NULL) + if((pThis->pszFName = MALLOC(sizeof(uchar) * (iLenName + 1))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */ @@ -1412,7 +1407,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - CHKmalloc(pThis->pszDir = malloc(sizeof(uchar) * iLenDir + 1)); + CHKmalloc(pThis->pszDir = MALLOC(sizeof(uchar) * (iLenDir + 1))); memcpy(pThis->pszDir, pszDir, iLenDir + 1); /* always think about the \0! */ pThis->lenDir = iLenDir; @@ -1510,6 +1505,46 @@ finalize_it: } +/* duplicate a stream object excluding dynamic properties. This function is + * primarily meant to provide a duplicate that later on can be used to access + * the data. This is needed, for example, for a restart of the disk queue. + * Note that ConstructFinalize() is NOT called. So our caller may change some + * properties before finalizing things. + * rgerhards, 2009-05-26 + */ +rsRetVal +strmDup(strm_t *pThis, strm_t **ppNew) +{ + strm_t *pNew = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + assert(ppNew != NULL); + + CHKiRet(strmConstruct(&pNew)); + pNew->sType = pThis->sType; + pNew->iCurrFNum = pThis->iCurrFNum; + CHKmalloc(pNew->pszFName = ustrdup(pThis->pszFName)); + pNew->lenFName = pThis->lenFName; + CHKmalloc(pNew->pszDir = ustrdup(pThis->pszDir)); + pNew->lenDir = pThis->lenDir; + pNew->tOperationsMode = pThis->tOperationsMode; + pNew->tOpenMode = pThis->tOpenMode; + pNew->iMaxFileSize = pThis->iMaxFileSize; + pNew->iMaxFiles = pThis->iMaxFiles; + pNew->iFileNumDigits = pThis->iFileNumDigits; + pNew->bDeleteOnClose = pThis->bDeleteOnClose; + pNew->iCurrOffs = pThis->iCurrOffs; + + *ppNew = pNew; + pNew = NULL; + +finalize_it: + if(pNew != NULL) + strmDestruct(&pNew); + + RETiRet; +} /* set a user write-counter. This counter is initialized to zero and * receives the number of bytes written. It is accurate only after a @@ -1625,6 +1660,7 @@ CODESTARTobjQueryInterface(strm) pIf->RecordEnd = strmRecordEnd; pIf->Serialize = strmSerialize; pIf->GetCurrOffset = strmGetCurrOffset; + pIf->Dup = strmDup; pIf->SetWCntr = strmSetWCntr; /* set methods */ pIf->SetbDeleteOnClose = strmSetbDeleteOnClose; diff --git a/runtime/stream.h b/runtime/stream.h index 369d5a0f..37e9d570 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -102,12 +102,12 @@ typedef struct strm_s { int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ - bool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ + sbool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ - bool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ - bool bSync; /* sync this file after every write? */ + sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ + sbool bSync; /* sync this file after every write? */ size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; @@ -118,13 +118,13 @@ typedef struct strm_s { size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ - bool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + sbool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; /* support for async flush procesing */ - bool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ - bool bStopWriter; /* shall writer thread terminate? */ - bool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ + sbool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ + sbool bStopWriter; /* shall writer thread terminate? */ + sbool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ int iFlushInterval; /* flush in which interval - 0, no flushing */ apc_id_t apcID; /* id of current Apc request (used for cancelling) */ pthread_mutex_t mut;/* mutex for flush in async mode */ @@ -143,7 +143,7 @@ typedef struct strm_s { /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ - bool bIsTTY; /* is this a tty file? */ + sbool bIsTTY; /* is this a tty file? */ } strm_t; @@ -169,6 +169,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm); rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); + rsRetVal (*Dup)(strm_t *pThis, strm_t **ppNew); INTERFACEpropSetMeth(strm, bDeleteOnClose, int); INTERFACEpropSetMeth(strm, iMaxFileSize, int); INTERFACEpropSetMeth(strm, iMaxFiles, int); @@ -183,7 +184,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, iFlushInterval, int); INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); ENDinterface(strm) -#define strmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/strgen.c b/runtime/strgen.c new file mode 100644 index 00000000..46be1236 --- /dev/null +++ b/runtime/strgen.c @@ -0,0 +1,279 @@ +/* strgen.c + * Module to handle string generators. These are C modules that receive + * the message object and return a custom-built string. The primary purpose + * for their existance is performance -- they do the same as template strings, but + * potentially faster (if well implmented). + * + * Module begun 2010-06-01 by Rainer Gerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include "rsyslog.h" +#include "msg.h" +#include "obj.h" +#include "errmsg.h" +#include "strgen.h" +#include "ruleset.h" +#include "unicode-helper.h" +#include "cfsysline.h" + +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(ruleset) + +/* static data */ + +/* config data */ + +/* This is the list of all strgens known to us. + * This is also used to unload all modules on shutdown. + */ +strgenList_t *pStrgenLstRoot = NULL; + + +/* intialize (but NOT allocate) a strgen list. Primarily meant as a hook + * which can be used to extend the list in the future. So far, just sets + * it to NULL. + */ +static rsRetVal +InitStrgenList(strgenList_t **pListRoot) +{ + *pListRoot = NULL; + return RS_RET_OK; +} + + +/* destruct a strgen list. The list elements are destroyed, but the strgen objects + * themselves are not modified. (That is done at a late stage during rsyslogd + * shutdown and need not be considered here.) + */ +static rsRetVal +DestructStrgenList(strgenList_t **ppListRoot) +{ + strgenList_t *pStrgenLst; + strgenList_t *pStrgenLstDel; + + pStrgenLst = *ppListRoot; + while(pStrgenLst != NULL) { + pStrgenLstDel = pStrgenLst; + pStrgenLst = pStrgenLst->pNext; + free(pStrgenLstDel); + } + *ppListRoot = NULL; + return RS_RET_OK; +} + + +/* Add a strgen to the list. We use a VERY simple and ineffcient algorithm, + * but it is employed only for a few milliseconds during config processing. So + * I prefer to keep it very simple and with simple data structures. Unfortunately, + * we need to preserve the order, but I don't like to add a tail pointer as that + * would require a container object. So I do the extra work to skip to the tail + * when adding elements... + */ +static rsRetVal +AddStrgenToList(strgenList_t **ppListRoot, strgen_t *pStrgen) +{ + strgenList_t *pThis; + strgenList_t *pTail; + DEFiRet; + + CHKmalloc(pThis = MALLOC(sizeof(strgenList_t))); + pThis->pStrgen = pStrgen; + pThis->pNext = NULL; + + if(*ppListRoot == NULL) { + pThis->pNext = *ppListRoot; + *ppListRoot = pThis; + } else { + /* find tail first */ + for(pTail = *ppListRoot ; pTail->pNext != NULL ; pTail = pTail->pNext) + /* just search, do nothing else */; + /* add at tail */ + pTail->pNext = pThis; + } + +finalize_it: + RETiRet; +} + + +/* find a strgen based on the provided name */ +static rsRetVal +FindStrgen(strgen_t **ppStrgen, uchar *pName) +{ + strgenList_t *pThis; + DEFiRet; + + for(pThis = pStrgenLstRoot ; pThis != NULL ; pThis = pThis->pNext) { + if(ustrcmp(pThis->pStrgen->pName, pName) == 0) { + *ppStrgen = pThis->pStrgen; + FINALIZE; /* found it, iRet still eq. OK! */ + } + } + + iRet = RS_RET_PARSER_NOT_FOUND; + +finalize_it: + RETiRet; +} + + +/* --- END helper functions for strgen list handling --- */ + + +BEGINobjConstruct(strgen) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(strgen) + +/* ConstructionFinalizer. The most important chore is to add the strgen object + * to our global list of available strgens. + * rgerhards, 2009-11-03 + */ +rsRetVal strgenConstructFinalize(strgen_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strgen); + CHKiRet(AddStrgenToList(&pStrgenLstRoot, pThis)); + DBGPRINTF("Strgen '%s' added to list of available strgens.\n", pThis->pName); + +finalize_it: + RETiRet; +} + +BEGINobjDestruct(strgen) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strgen) + dbgprintf("destructing strgen '%s'\n", pThis->pName); + free(pThis->pName); +ENDobjDestruct(strgen) + +/* set the strgen name - string is copied over, call can continue to use it, + * but must free it if desired. + */ +static rsRetVal +SetName(strgen_t *pThis, uchar *name) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strgen); + assert(name != NULL); + + if(pThis->pName != NULL) { + free(pThis->pName); + pThis->pName = NULL; + } + + CHKmalloc(pThis->pName = ustrdup(name)); + +finalize_it: + RETiRet; +} + + +/* set a pointer to "our" module. Note that no module + * pointer must already be set. + */ +static rsRetVal +SetModPtr(strgen_t *pThis, modInfo_t *pMod) +{ + ISOBJ_TYPE_assert(pThis, strgen); + assert(pMod != NULL); + assert(pThis->pModule == NULL); + pThis->pModule = pMod; + return RS_RET_OK; +} + + +/* queryInterface function-- rgerhards, 2009-11-03 + */ +BEGINobjQueryInterface(strgen) +CODESTARTobjQueryInterface(strgen) + if(pIf->ifVersion != strgenCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = strgenConstruct; + pIf->ConstructFinalize = strgenConstructFinalize; + pIf->Destruct = strgenDestruct; + pIf->SetName = SetName; + pIf->SetModPtr = SetModPtr; + pIf->InitStrgenList = InitStrgenList; + pIf->DestructStrgenList = DestructStrgenList; + pIf->AddStrgenToList = AddStrgenToList; + pIf->FindStrgen = FindStrgen; +finalize_it: +ENDobjQueryInterface(strgen) + + +/* This destroys the master strgenlist and all of its strgen entries. MUST only be + * done when the module is shut down. Strgen modules are NOT unloaded, rsyslog + * does that at a later stage for all dynamically loaded modules. + */ +static void +destroyMasterStrgenList(void) +{ + strgenList_t *pStrgenLst; + strgenList_t *pStrgenLstDel; + + pStrgenLst = pStrgenLstRoot; + while(pStrgenLst != NULL) { + strgenDestruct(&pStrgenLst->pStrgen); + pStrgenLstDel = pStrgenLst; + pStrgenLst = pStrgenLst->pNext; + free(pStrgenLstDel); + } +} + +/* Exit our class. + * rgerhards, 2009-11-04 + */ +BEGINObjClassExit(strgen, OBJ_IS_CORE_MODULE) /* class, version */ + destroyMasterStrgenList(); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDObjClassExit(strgen) + + +/* Initialize the strgen class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2009-11-02 + */ +BEGINObjClassInit(strgen, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + InitStrgenList(&pStrgenLstRoot); +ENDObjClassInit(strgen) + diff --git a/runtime/strgen.h b/runtime/strgen.h new file mode 100644 index 00000000..3819dccd --- /dev/null +++ b/runtime/strgen.h @@ -0,0 +1,60 @@ +/* header for strgen.c + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRGEN_H +#define INCLUDED_STRGEN_H + + +/* we create a small helper object, a list of strgens, that we can use to + * build a chain of them whereever this is needed. + */ +struct strgenList_s { + strgen_t *pStrgen; + strgenList_t *pNext; +}; + + +/* the strgen object, a dummy because we have only static methods */ +struct strgen_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *pName; /* name of this strgen */ + modInfo_t *pModule; /* pointer to strgen's module */ +}; + +/* interfaces */ +BEGINinterface(strgen) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(strgen_t **ppThis); + rsRetVal (*ConstructFinalize)(strgen_t *pThis); + rsRetVal (*Destruct)(strgen_t **ppThis); + rsRetVal (*SetName)(strgen_t *pThis, uchar *name); + rsRetVal (*SetModPtr)(strgen_t *pThis, modInfo_t *pMod); + rsRetVal (*FindStrgen)(strgen_t **ppThis, uchar*name); + rsRetVal (*InitStrgenList)(strgenList_t **pListRoot); + rsRetVal (*DestructStrgenList)(strgenList_t **pListRoot); + rsRetVal (*AddStrgenToList)(strgenList_t **pListRoot, strgen_t *pStrgen); +ENDinterface(strgen) +#define strgenCURR_IF_VERSION 1 /* increment whenever you change the interface above! */ + + +/* prototypes */ +PROTOTYPEObj(strgen); + +#endif /* #ifndef INCLUDED_STRGEN_H */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 93995b38..ccf115c1 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -90,7 +90,7 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -119,7 +119,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = pFrom->iStrLen; - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -267,7 +267,7 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) pThis->pszBuf = NULL; /* now save the new value */ - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); return RS_RET_OUT_OF_MEMORY; } @@ -315,7 +315,7 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) if(pThis->pBuf != NULL) if(pThis->pszBuf == NULL) { /* we do not yet have a usable sz version - so create it... */ - if((pThis->pszBuf = malloc((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { + if((pThis->pszBuf = MALLOC((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { /* TODO: think about what to do - so far, I have no bright * idea... rgerhards 2005-09-07 */ @@ -369,7 +369,7 @@ rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - CHKmalloc(pRetBuf = malloc(sizeof(uchar))); + CHKmalloc(pRetBuf = MALLOC(sizeof(uchar))); *pRetBuf = '\0'; } else { pRetBuf = NULL; diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index 3dc53a97..a122ca8a 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -165,7 +165,7 @@ addNewLstnPort(strmsrv_t *pThis, uchar *pszPort) ISOBJ_TYPE_assert(pThis, strmsrv); /* create entry */ - CHKmalloc(pEntry = malloc(sizeof(strmLstnPortList_t))); + CHKmalloc(pEntry = MALLOC(sizeof(strmLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); diff --git a/runtime/sync.c b/runtime/sync.c index a3053e28..15d5c80f 100644 --- a/runtime/sync.c +++ b/runtime/sync.c @@ -28,12 +28,13 @@ #include "rsyslog.h" #include "sync.h" +#include "debug.h" void SyncObjInit(pthread_mutex_t **mut) { - *mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + *mut = (pthread_mutex_t *) MALLOC(sizeof (pthread_mutex_t)); pthread_mutex_init(*mut, NULL); } diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h index 4a26f993..fcebd985 100644 --- a/runtime/syslogd-types.h +++ b/runtime/syslogd-types.h @@ -56,7 +56,10 @@ * applications I do not yet envision. -- rgerhards, 2007-07-24 */ typedef enum _syslogFeature { - sFEATURERepeatedMsgReduction = 1 + sFEATURERepeatedMsgReduction = 1, /* for output modules */ + sFEATURENonCancelInputTermination = 2, /* for input modules */ + sFEATUREAutomaticSanitazion = 3, /* for parser modules */ + sFEATUREAutomaticPRIParsing = 4 /* for parser modules */ } syslogFeature; /* we define our own facility and severities */ diff --git a/runtime/wti.c b/runtime/wti.c index 90bb14ed..9343f5c5 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -39,10 +39,6 @@ #include <pthread.h> #include <errno.h> -#ifdef OS_SOLARIS -# include <sched.h> -#endif - #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" @@ -75,92 +71,72 @@ wtiGetDbgHdr(wti_t *pThis) } -/* get the current worker state. For simplicity and speed, we have - * NOT used our regular calling interface this time. I hope that won't - * bite in the long term... -- rgerhards, 2008-01-17 - * TODO: may be performance optimized by atomic operations +/* return the current worker processing state. For the sake of + * simplicity, we do not use the iRet interface. -- rgerhards, 2009-07-17 */ -qWrkCmd_t -wtiGetState(wti_t *pThis, int bLockMutex) +sbool +wtiGetState(wti_t *pThis) { - DEFVARS_mutexProtection; - qWrkCmd_t tCmd; + return ATOMIC_FETCH_32BIT(&pThis->bIsRunning, &pThis->mutIsRunning); +} - BEGINfunc - ISOBJ_TYPE_assert(pThis, wti); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - tCmd = pThis->tCurrCmd; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); +/* Set this thread to "always running" state (can not be unset) + * rgerhards, 2009-07-20 + */ +rsRetVal +wtiSetAlwaysRunning(wti_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, wti); + pThis->bAlwaysRunning = TRUE; + return RS_RET_OK; +} - ENDfunc - return tCmd; +/* Set status (thread is running or not), actually an property of + * use for wtp, but we need to have it per thread instance (thus it + * is inside wti). -- rgerhards, 2009-07-17 + */ +rsRetVal +wtiSetState(wti_t *pThis, sbool bNewVal) +{ + ISOBJ_TYPE_assert(pThis, wti); + if(bNewVal) { + ATOMIC_STORE_1_TO_INT(&pThis->bIsRunning, &pThis->mutIsRunning); + } else { + ATOMIC_STORE_0_TO_INT(&pThis->bIsRunning, &pThis->mutIsRunning); + } + return RS_RET_OK; } -/* send a command to a specific thread - * bActiveOnly specifies if the command should be sent only when the worker is - * in an active state. -- rgerhards, 2008-01-20 +/* advise all workers to start by interrupting them. That should unblock all srSleep() + * calls. */ rsRetVal -wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) +wtiWakeupThrd(wti_t *pThis) { DEFiRet; - qWrkCmd_t tCurrCmd; - DEFVARS_mutexProtection; ISOBJ_TYPE_assert(pThis, wti); - assert(tCmd <= eWRKTHRD_SHUTDOWN_IMMEDIATE); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - - tCurrCmd = pThis->tCurrCmd; - /* all worker states must be followed sequentially, only termination can be set in any state */ - if( (bActiveOnly && (tCurrCmd < eWRKTHRD_RUN_CREATED)) - || (tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) { - DBGPRINTF("%s: command %d can not be accepted in current %d processing state - ignored\n", - wtiGetDbgHdr(pThis), tCmd, tCurrCmd); - } else { - DBGPRINTF("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd); - /* we could replace this with a simple if, but we leave the switch in in case we need - * to add something at a later stage. -- rgerhards, 2008-09-30 - */ - switch(tCmd) { - case eWRKTHRD_TERMINATING: - /* TODO: re-enable meaningful debug msg! (via function callback?) - dbgprintf("%s: thread terminating with %d entries left in queue, %d workers running.\n", - wtiGetDbgHdr(pThis->pQueue), pThis->pQueue->iQueueSize, - pThis->pQueue->iCurNumWrkThrd); - */ - pthread_cond_signal(&pThis->condExitDone); - dbgprintf("%s: worker terminating\n", wtiGetDbgHdr(pThis)); - break; - /* these cases just to satisfy the compiler, we do (yet) not act an them: */ - case eWRKTHRD_RUNNING: - case eWRKTHRD_STOPPED: - case eWRKTHRD_RUN_CREATED: - case eWRKTHRD_RUN_INIT: - case eWRKTHRD_SHUTDOWN: - case eWRKTHRD_SHUTDOWN_IMMEDIATE: - /* DO NOTHING */ - break; - } - /* apply the new state */ - unsigned val = ATOMIC_CAS_VAL((int*)&pThis->tCurrCmd, tCurrCmd, tCmd, &pThis->mutCurrCmd); - if(val != tCurrCmd) { - DBGPRINTF("wtiSetState PROBLEM, tCurrCmd %d overwritten with %d, wanted to set %d\n", tCurrCmd, val, tCmd); - } + if(wtiGetState(pThis)) { + /* we first try the cooperative "cancel" interface */ + pthread_kill(pThis->thrdID, SIGTTIN); + dbgprintf("sent SIGTTIN to worker thread 0x%x\n", (unsigned) pThis->thrdID); } - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); RETiRet; } -/* Cancel the thread. If the thread is already cancelled or terminated, - * we do not again cancel it. But it is save and legal to call wtiCancelThrd() in - * such situations. +/* Cancel the thread. If the thread is not running. But it is save and legal to + * call wtiCancelThrd() in such situations. This function only returns when the + * thread has terminated. Else we may get race conditions all over the code... + * Note that when waiting for the thread to terminate, we do a busy wait, checking + * progress every 10ms. It is very unlikely that we will ever cancel a thread + * and, if so, it will only happen at the end of the rsyslog run. So doing this + * kind of non-optimal wait is considered preferable over using condition variables. * rgerhards, 2008-02-26 */ rsRetVal @@ -170,19 +146,24 @@ wtiCancelThrd(wti_t *pThis) ISOBJ_TYPE_assert(pThis, wti); - d_pthread_mutex_lock(&pThis->mut); - wtiProcessThrdChanges(pThis, MUTEX_ALREADY_LOCKED); /* process state change, so that we have current state vars */ + if(wtiGetState(pThis)) { + /* we first try the cooperative "cancel" interface */ + pthread_kill(pThis->thrdID, SIGTTIN); + dbgprintf("sent SIGTTIN to worker thread 0x%x, giving it a chance to terminate\n", (unsigned) pThis->thrdID); + srSleep(0, 10000); + } - if(pThis->tCurrCmd >= eWRKTHRD_TERMINATING) { - dbgoprint((obj_t*) pThis, "canceling worker thread, curr stat %d\n", pThis->tCurrCmd); + if(wtiGetState(pThis)) { + dbgprintf("cooperative worker termination failed, using cancellation...\n"); + dbgoprint((obj_t*) pThis, "canceling worker thread\n"); pthread_cancel(pThis->thrdID); - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - wtpSetThrdStateChanged(pThis->pWtp, 1); /* indicate change, so harverster will be called */ + /* now wait until the thread terminates... */ + while(wtiGetState(pThis)) { + srSleep(0, 10000); + } } - d_pthread_mutex_unlock(&pThis->mut); - RETiRet; } @@ -190,26 +171,9 @@ wtiCancelThrd(wti_t *pThis) /* Destructor */ BEGINobjDestruct(wti) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(wti) - /* if we reach this point, we must make sure the associated worker has terminated. It is - * the callers duty to make sure the worker already knows it shall terminate. - * TODO: is it *really* the caller's duty? ...mmmhhhh.... smells bad... rgerhards, 2008-01-25 - */ - wtiProcessThrdChanges(pThis, LOCK_MUTEX); /* process state change one last time */ - - d_pthread_mutex_lock(&pThis->mut); - if(wtiGetState(pThis, MUTEX_ALREADY_LOCKED) != eWRKTHRD_STOPPED) { - dbgprintf("%s: WARNING: worker %p shall be destructed but is still running (might be OK) - joining it\n", - wtiGetDbgHdr(pThis), pThis); - /* let's hope the caller actually instructed it to shutdown... */ - pthread_cond_wait(&pThis->condExitDone, &pThis->mut); - wtiJoinThrd(pThis); - } - d_pthread_mutex_unlock(&pThis->mut); - /* actual destruction */ - pthread_cond_destroy(&pThis->condExitDone); - pthread_mutex_destroy(&pThis->mut); - DESTROY_ATOMIC_HELPER_MUT(pThis->mutCurrCmd); + batchFree(&pThis->batch); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutIsRunning); free(pThis->pszDbgHdr); ENDobjDestruct(wti) @@ -218,9 +182,7 @@ ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ - pthread_cond_init(&pThis->condExitDone, NULL); - pthread_mutex_init(&pThis->mut, NULL); - INIT_ATOMIC_HELPER_MUT(pThis->mutCurrCmd); + INIT_ATOMIC_HELPER_MUT(pThis->mutIsRunning); ENDobjConstruct(wti) @@ -231,81 +193,28 @@ rsRetVal wtiConstructFinalize(wti_t *pThis) { DEFiRet; + int iDeqBatchSize; ISOBJ_TYPE_assert(pThis, wti); dbgprintf("%s: finalizing construction of worker instance data\n", wtiGetDbgHdr(pThis)); - /* initialize our thread instance descriptor */ - pThis->pUsrp = NULL; - pThis->tCurrCmd = eWRKTHRD_STOPPED; - - RETiRet; -} - - -/* join a specific worker thread - * we do not lock the mutex, because join will sync anyways... - */ -rsRetVal -wtiJoinThrd(wti_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, wti); - dbgprintf("waiting for worker %s termination, current state %d\n", wtiGetDbgHdr(pThis), pThis->tCurrCmd); - if (pThis->thrdID == 0) { - dbgprintf("worker %s was already stopped\n", wtiGetDbgHdr(pThis)); - } else { - pthread_join(pThis->thrdID, NULL); - wtiSetState(pThis, eWRKTHRD_STOPPED, 0, MUTEX_ALREADY_LOCKED); /* back to virgin... */ - pThis->thrdID = 0; /* invalidate the thread ID so that we do not accidently find reused ones */ - dbgprintf("worker %s has stopped\n", wtiGetDbgHdr(pThis)); - } - - RETiRet; -} - -/* check if we had a worker thread changes and, if so, act - * on it. At a minimum, terminated threads are harvested (joined). - */ -rsRetVal -wtiProcessThrdChanges(wti_t *pThis, int bLockMutex) -{ - DEFiRet; - DEFVARS_mutexProtection; + /* initialize our thread instance descriptor (no concurrency here) */ + pThis->bIsRunning = FALSE; - ISOBJ_TYPE_assert(pThis, wti); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - switch(pThis->tCurrCmd) { - case eWRKTHRD_TERMINATING: - /* we need to at least temporarily release the mutex, because otherwise - * we may deadlock with the thread we intend to join (it aquires the mutex - * during termination processing). -- rgerhards, 2008-02-26 - */ - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - iRet = wtiJoinThrd(pThis); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - break; - /* these cases just to satisfy the compiler, we do not act an them: */ - case eWRKTHRD_STOPPED: - case eWRKTHRD_RUN_CREATED: - case eWRKTHRD_RUN_INIT: - case eWRKTHRD_RUNNING: - case eWRKTHRD_SHUTDOWN: - case eWRKTHRD_SHUTDOWN_IMMEDIATE: - /* DO NOTHING */ - break; - } - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ + CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize)); + CHKiRet(batchInit(&pThis->batch, iDeqBatchSize)); +finalize_it: RETiRet; } /* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. + * Most importantly, it must bring back the batch into a consistent state. + * Keep in mind that cancellation is disabled if we run into + * the cancel cleanup handler (and have been cancelled). * rgerhards, 2008-01-16 */ static void @@ -313,7 +222,6 @@ wtiWorkerCancelCleanup(void *arg) { wti_t *pThis = (wti_t*) arg; wtp_t *pWtp; - int iCancelStateSave; BEGINfunc ISOBJ_TYPE_assert(pThis, wti); @@ -321,102 +229,108 @@ wtiWorkerCancelCleanup(void *arg) ISOBJ_TYPE_assert(pWtp, wtp); DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); + pWtp->pfObjProcessed(pWtp->pUsr, pThis); + DBGPRINTF("%s: done cancelation cleanup handler.\n", wtiGetDbgHdr(pThis)); - /* call user supplied handler (that one e.g. requeues the element) */ - pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp); + ENDfunc +} - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pWtp->mut); - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - wtpSetThrdStateChanged(pWtp, 1); /* indicate change, so harverster will be called */ - d_pthread_mutex_unlock(&pWtp->mut); - pthread_setcancelstate(iCancelStateSave, NULL); +/* wait for queue to become non-empty or timeout + * helper to wtiWorker. Note the the predicate is + * re-tested by the caller, so it is OK to NOT do it here. + * rgerhards, 2009-05-20 + */ +static inline void +doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) +{ + struct timespec t; + + BEGINfunc + DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); + + if(pThis->bAlwaysRunning) { + /* never shut down any started worker */ + d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); + } else { + timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ + if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { + DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); + *pbInactivityTOOccured = 1; /* indicate we had a timeout */ + } + } + dbgoprint((obj_t*) pThis, "worker awoke from idle processing\n"); ENDfunc } -/* generic worker thread framework +/* generic worker thread framework. Note that we prohibit cancellation + * during almost all times, because it can have very undesired side effects. + * However, we may need to cancel a thread if the consumer blocks for too + * long (during shutdown). So what we do is block cancellation, and every + * consumer must enable it during the periods where it is safe. */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal wtiWorker(wti_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; - struct timespec t; wtp_t *pWtp; /* our worker thread pool */ int bInactivityTOOccured = 0; + rsRetVal localRet; + rsRetVal terminateRet; + int iCancelStateSave; + DEFiRet; ISOBJ_TYPE_assert(pThis, wti); pWtp = pThis->pWtp; /* shortcut */ ISOBJ_TYPE_assert(pWtp, wtp); dbgSetThrdName(pThis->pszDbgHdr); - pThis->pUsrp = NULL; pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - pWtp->pfOnWorkerStartup(pWtp->pUsr); - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); /* now we have our identity, on to real processing */ while(1) { /* loop will be broken below - need to do mutex locks */ - /* process any pending thread requests */ - wtpProcessThrdChanges(pWtp); - - /* if we have a rate-limiter set for this worker pool, let's call it. Please - * keep in mind that the rate-limiter may hold us for an extended period - * of time. -- rgerhards, 2008-04-02 - */ - if(pWtp->pfRateLimiter != NULL) { + if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */ pWtp->pfRateLimiter(pWtp->pUsr); } - wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - - if( (bInactivityTOOccured && pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) - || wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED)) { - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); - break; /* end worker thread run */ + d_pthread_mutex_lock(pWtp->pmutUsr); + + /* first check if we are in shutdown process (but evaluate a bit later) */ + terminateRet = wtpChkStopWrkr(pWtp, MUTEX_ALREADY_LOCKED); + if(terminateRet == RS_RET_TERMINATE_NOW) { + /* we now need to free the old batch */ + localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis); + dbgoprint((obj_t*) pThis, "terminating worker because of TERMINATE_NOW mode, del iRet %d\n", + localRet); + d_pthread_mutex_unlock(pWtp->pmutUsr); + break; } - bInactivityTOOccured = 0; /* reset for next run */ - /* if we reach this point, we are still protected by the mutex */ - - if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) { - DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); - pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); - - if(pWtp->toWrkShutdown == -1) { - /* never shut down any started worker */ - d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); - } else { - timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ - if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { - DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); - bInactivityTOOccured = 1; /* indicate we had a timeout */ - } + /* try to execute and process whatever we have */ + /* Note that this function releases and re-aquires the mutex. The returned + * information on idle state must be processed before releasing the mutex again. + */ + localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); + + if(localRet == RS_RET_IDLE) { + if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { + d_pthread_mutex_unlock(pWtp->pmutUsr); + break; /* end of loop */ } - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); + d_pthread_mutex_unlock(pWtp->pmutUsr); continue; /* request next iteration */ } - /* if we reach this point, we have a non-empty queue (and are still protected by mutex) */ - pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); + d_pthread_mutex_unlock(pWtp->pmutUsr); + + bInactivityTOOccured = 0; /* reset for next run */ } /* indicate termination */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pThis->mut); pthread_cleanup_pop(0); /* remove cleanup handler */ - - pWtp->pfOnWorkerShutdown(pWtp->pUsr); - - wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - wtpSetThrdStateChanged(pWtp, 1); /* indicate change, so harverster will be called */ - d_pthread_mutex_unlock(&pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); RETiRet; @@ -445,10 +359,9 @@ wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg) if(pThis->pszDbgHdr != NULL) { free(pThis->pszDbgHdr); - pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ @@ -479,6 +392,5 @@ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wti) -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/wti.h b/runtime/wti.h index d81672f3..51ece4ef 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -1,6 +1,6 @@ /* Definition of the worker thread instance (wti) class. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 by Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -27,22 +27,20 @@ #include <pthread.h> #include "wtp.h" #include "obj.h" +#include "batch.h" + /* the worker thread instance class */ -typedef struct wti_s { +struct wti_s { BEGINobjInstance; - pthread_t thrdID; /* thread ID */ - qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ + pthread_t thrdID; /* thread ID */ + int bIsRunning; /* is this thread currently running? (must be int for atomic op!) */ + sbool bAlwaysRunning; /* should this thread always run? */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ - pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */ - pthread_mutex_t mut; - bool bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ + batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ uchar *pszDbgHdr; /* header string for debug messages */ - DEF_ATOMIC_HELPER_MUT(mutCurrCmd); -} wti_t; - -/* some symbolic constants for easier reference */ + DEF_ATOMIC_HELPER_MUT(mutIsRunning); +}; /* prototypes */ @@ -50,12 +48,12 @@ rsRetVal wtiConstruct(wti_t **ppThis); rsRetVal wtiConstructFinalize(wti_t *pThis); rsRetVal wtiDestruct(wti_t **ppThis); rsRetVal wtiWorker(wti_t *pThis); -rsRetVal wtiProcessThrdChanges(wti_t *pThis, int bLockMutex); rsRetVal wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg); -rsRetVal wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex); -rsRetVal wtiJoinThrd(wti_t *pThis); rsRetVal wtiCancelThrd(wti_t *pThis); -qWrkCmd_t wtiGetState(wti_t *pThis, int bLockMutex); +rsRetVal wtiSetAlwaysRunning(wti_t *pThis); +rsRetVal wtiSetState(wti_t *pThis, sbool bNew); +rsRetVal wtiWakeupThrd(wti_t *pThis); +sbool wtiGetState(wti_t *pThis); PROTOTYPEObjClassInit(wti); PROTOTYPEpropSetMeth(wti, pszDbgHdr, uchar*); PROTOTYPEpropSetMeth(wti, pWtp, wtp_t*); diff --git a/runtime/wtp.c b/runtime/wtp.c index fff37c2f..ece80911 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -8,7 +8,7 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -44,9 +44,10 @@ # include <sys/prctl.h> #endif -#ifdef OS_SOLARIS -# include <sched.h> -#endif +/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 +//#ifdef OS_SOLARIS +//# include <sched.h> +//#endif #include "rsyslog.h" #include "stringbuf.h" @@ -82,22 +83,21 @@ wtpGetDbgHdr(wtp_t *pThis) /* Not implemented dummy function for constructor */ -static rsRetVal NotImplementedDummy() { return RS_RET_OK; } +static rsRetVal NotImplementedDummy() { return RS_RET_NOT_IMPLEMENTED; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ - pthread_mutex_init(&pThis->mut, NULL); - pthread_mutex_init(&pThis->mutThrdShutdwn, NULL); + pthread_mutex_init(&pThis->mutWtp, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); + pthread_attr_init(&pThis->attrThrd); + pthread_attr_setdetachstate(&pThis->attrThrd, PTHREAD_CREATE_DETACHED); /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; - pThis->pfIsIdle = NotImplementedDummy; + pThis->pfGetDeqBatchSize = NotImplementedDummy; pThis->pfDoWork = NotImplementedDummy; - pThis->pfOnIdle = NotImplementedDummy; - pThis->pfOnWorkerCancel = NotImplementedDummy; - pThis->pfOnWorkerStartup = NotImplementedDummy; - pThis->pfOnWorkerShutdown = NotImplementedDummy; - INIT_ATOMIC_HELPER_MUT(pThis->mutThrdStateChanged); + pThis->pfObjProcessed = NotImplementedDummy; + INIT_ATOMIC_HELPER_MUT(pThis->mutCurNumWrkThrd); + INIT_ATOMIC_HELPER_MUT(pThis->mutWtpState); ENDobjConstruct(wtp) @@ -115,13 +115,12 @@ wtpConstructFinalize(wtp_t *pThis) ISOBJ_TYPE_assert(pThis, wtp); - dbgprintf("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis)); + DBGPRINTF("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis)); /* alloc and construct workers - this can only be done in finalizer as we previously do * not know the max number of workers */ - if((pThis->pWrkr = malloc(sizeof(wti_t*) * pThis->iNumWorkerThreads)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - + CHKmalloc(pThis->pWrkr = MALLOC(sizeof(wti_t*) * pThis->iNumWorkerThreads)); + for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { CHKiRet(wtiConstruct(&pThis->pWrkr[i])); pWti = pThis->pWrkr[i]; @@ -141,8 +140,6 @@ finalize_it: BEGINobjDestruct(wtp) /* be sure to specify the object type also in END and CODESTART macros! */ int i; CODESTARTobjDestruct(wtp) - wtpProcessThrdChanges(pThis); /* process thread changes one last time */ - /* destruct workers */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) wtiDestruct(&pThis->pWrkr[i]); @@ -152,149 +149,70 @@ CODESTARTobjDestruct(wtp) /* actual destruction */ pthread_cond_destroy(&pThis->condThrdTrm); - pthread_mutex_destroy(&pThis->mut); - pthread_mutex_destroy(&pThis->mutThrdShutdwn); - DESTROY_ATOMIC_HELPER_MUT(pThis->mutThrdStateChanged); + pthread_mutex_destroy(&pThis->mutWtp); + pthread_attr_destroy(&pThis->attrThrd); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutCurNumWrkThrd); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutWtpState); free(pThis->pszDbgHdr); ENDobjDestruct(wtp) -/* wake up at least one worker thread. - * rgerhards, 2008-01-20 - */ -rsRetVal -wtpWakeupWrkr(wtp_t *pThis) -{ - DEFiRet; - - /* TODO; mutex? I think not needed, as we do not need predictable exec order -- rgerhards, 2008-01-28 */ - ISOBJ_TYPE_assert(pThis, wtp); - pthread_cond_signal(pThis->pcondBusy); - RETiRet; -} - -/* wake up all worker threads. - * rgerhards, 2008-01-16 - */ -rsRetVal -wtpWakeupAllWrkr(wtp_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, wtp); - pthread_cond_broadcast(pThis->pcondBusy); - RETiRet; -} - - -/* set the bThrdStateChanged in an atomic way. Note that - * val may only be 0 or 1. - */ -void -wtpSetThrdStateChanged(wtp_t *pThis, int val) -{ - if(val == 0) { - ATOMIC_STORE_0_TO_INT(&pThis->bThrdStateChanged, pThis->mutThrdStateChanged); - } else { - ATOMIC_STORE_1_TO_INT(&pThis->bThrdStateChanged, pThis->mutThrdStateChanged); - } -} - - -/* check if we had any worker thread changes and, if so, act - * on them. At a minimum, terminated threads are harvested (joined). - * This function MUST NEVER block on the queue mutex! - */ -rsRetVal -wtpProcessThrdChanges(wtp_t *pThis) -{ - DEFiRet; - int i; - - ISOBJ_TYPE_assert(pThis, wtp); - - if(pThis->bThrdStateChanged == 0) - FINALIZE; - - if(d_pthread_mutex_trylock(&(pThis->mutThrdShutdwn)) != 0) { - /* another thread is already in the loop */ - FINALIZE; - } - - /* Note: there is a left-over potential race condition below: - * pThis->bThrdStateChanged may be re-set by another thread while - * we work on it and thus the loop may terminate too early. However, - * there are no really bad effects from that so I perfer - for this - * version - to live with the problem as is. Not a good idea to - * introduce that large change into the stable branch without very - * good reason. -- rgerhards, 2009-04-02 - */ - do { - /* reset the change marker */ - wtpSetThrdStateChanged(pThis, 0); - /* go through all threads */ - for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - wtiProcessThrdChanges(pThis->pWrkr[i], LOCK_MUTEX); - } - /* restart if another change occured while we were processing the changes */ - } while(pThis->bThrdStateChanged != 0); - - d_pthread_mutex_unlock(&(pThis->mutThrdShutdwn)); - -finalize_it: - RETiRet; -} - - -/* Sent a specific state for the worker thread pool. - * rgerhards, 2008-01-21 +/* Sent a specific state for the worker thread pool. -- rgerhards, 2008-01-21 + * We do not need to do atomic instructions as set operations are only + * called when terminating the pool, and then in strict sequence. So we + * can never overwrite each other. On the other hand, it also doesn't + * matter if the read operation obtains an older value, as we then simply + * do one more iteration, what is perfectly legal (during shutdown + * they are awoken in any case). -- rgerhards, 2009-07-20 */ rsRetVal wtpSetState(wtp_t *pThis, wtpState_t iNewState) { - DEFiRet; - ISOBJ_TYPE_assert(pThis, wtp); - pThis->wtpState = iNewState; - /* TODO: must wakeup workers? seen to be not needed -- rgerhards, 2008-01-28 */ - - RETiRet; + pThis->wtpState = iNewState; // TODO: do we need a mutex here? 2010-04-26 + return RS_RET_OK; } /* check if the worker shall shutdown (1 = yes, 0 = no) - * TODO: check if we can use atomic operations to enhance performance * Note: there may be two mutexes locked, the bLockUsrMutex is the one in our "user" * (e.g. the queue clas) * rgerhards, 2008-01-21 */ rsRetVal -wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex) +wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex) { DEFiRet; - DEFVARS_mutexProtection; + wtpState_t wtpState; ISOBJ_TYPE_assert(pThis, wtp); + /* we need a consistent value, but it doesn't really matter if it is changed + * right after the fetch - then we simply do one more iteration in the worker + */ + wtpState = (wtpState_t) ATOMIC_FETCH_32BIT((int*)&pThis->wtpState, &pThis->mutWtpState); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - if( (pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) - || ((pThis->wtpState == wtpState_SHUTDOWN) && pThis->pfIsIdle(pThis->pUsr, bLockUsrMutex))) - iRet = RS_RET_TERMINATE_NOW; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + if(wtpState == wtpState_SHUTDOWN_IMMEDIATE) { + ABORT_FINALIZE(RS_RET_TERMINATE_NOW); + } else if(wtpState == wtpState_SHUTDOWN) { + ABORT_FINALIZE(RS_RET_TERMINATE_WHEN_IDLE); + } /* try customer handler if one was set and we do not yet have a definite result */ - if(iRet == RS_RET_OK && pThis->pfChkStopWrkr != NULL) { + if(pThis->pfChkStopWrkr != NULL) { iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex); } +finalize_it: RETiRet; } #pragma GCC diagnostic ignored "-Wempty-body" /* Send a shutdown command to all workers and see if they terminate. - * A timeout may be specified. + * A timeout may be specified. This function may also be called with + * the current number of workers being 0, in which case it does not + * shut down any worker. * rgerhards, 2008-01-14 */ rsRetVal @@ -302,72 +220,50 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout { DEFiRet; int bTimedOut; - int iCancelStateSave; + int i; ISOBJ_TYPE_assert(pThis, wtp); + /* lock mutex to prevent races (may otherwise happen during idle processing and such...) */ + d_pthread_mutex_lock(pThis->pmutUsr); wtpSetState(pThis, tShutdownCmd); - wtpWakeupAllWrkr(pThis); + pthread_cond_broadcast(pThis->pcondBusy); /* wake up all workers */ + /* awake workers in retry loop */ + for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { + wtiWakeupThrd(pThis->pWrkr[i]); + } + d_pthread_mutex_unlock(pThis->pmutUsr); - /* see if we need to harvest (join) any terminated threads (even in timeout case, - * some may have terminated... - */ - wtpProcessThrdChanges(pThis); - - /* and wait for their termination */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pThis->mut); - pthread_cleanup_push(mutexCancelCleanup, &pThis->mut); - pthread_setcancelstate(iCancelStateSave, NULL); + /* wait for worker thread termination */ + d_pthread_mutex_lock(&pThis->mutWtp); + pthread_cleanup_push(mutexCancelCleanup, &pThis->mutWtp); bTimedOut = 0; while(pThis->iCurNumWrkThrd > 0 && !bTimedOut) { - dbgprintf("%s: waiting %ldms on worker thread termination, %d still running\n", - wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), pThis->iCurNumWrkThrd); + DBGPRINTF("%s: waiting %ldms on worker thread termination, %d still running\n", + wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), + ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd)); - if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mut, ptTimeout) != 0) { - dbgprintf("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis)); + if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mutWtp, ptTimeout) != 0) { + DBGPRINTF("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis)); bTimedOut = 1; /* we exit the loop on timeout */ } + + /* awake workers in retry loop */ + for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { + wtiWakeupThrd(pThis->pWrkr[i]); + } + } pthread_cleanup_pop(1); if(bTimedOut) iRet = RS_RET_TIMED_OUT; - /* see if we need to harvest (join) any terminated threads (even in timeout case, - * some may have terminated... - */ - wtpProcessThrdChanges(pThis); - RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" -/* indicate that a thread has terminated and awake anyone waiting on it - * rgerhards, 2008-01-23 - */ -rsRetVal wtpSignalWrkrTermination(wtp_t *pThis) -{ - DEFiRet; - /* I leave the mutex code here out as it gives us deadlocks. I think it is not really - * needed and we are on the safe side. I leave this comment in if practice proves us - * wrong. The whole thing should be removed after half a year or year if we see there - * actually is no issue (or revisit it from a theoretical POV). - * rgerhards, 2008-01-28 - * revisited 2008-09-30, still a bit unclear, leave in - */ - /*TODO: mutex or not mutex, that's the question ;)DEFVARS_mutexProtection;*/ - - ISOBJ_TYPE_assert(pThis, wtp); - - /*BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX);*/ - pthread_cond_signal(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ - /*END_MTX_PROTECTED_OPERATIONS(&pThis->mut);*/ - RETiRet; -} - - /* Unconditionally cancel all running worker threads. * rgerhards, 2008-01-14 */ @@ -379,12 +275,8 @@ wtpCancelAll(wtp_t *pThis) ISOBJ_TYPE_assert(pThis, wtp); - /* process any pending thread requests so that we know who actually is still running */ - wtpProcessThrdChanges(pThis); - /* go through all workers and cancel those that are active */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - dbgprintf("%s: try canceling worker thread %d\n", wtpGetDbgHdr(pThis), i); wtiCancelThrd(pThis->pWrkr[i]); } @@ -392,40 +284,57 @@ wtpCancelAll(wtp_t *pThis) } - -/* Set the Inactivity Guard - * rgerhards, 2008-01-21 +/* this function contains shared code for both regular worker shutdown as + * well as shutdown via cancellation. We can not simply use pthread_cleanup_pop(1) + * as this introduces a race in the debug system (RETiRet system). + * rgerhards, 2009-10-26 */ -rsRetVal -wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex) +static inline void +wtpWrkrExecCleanup(wti_t *pWti) { - DEFiRet; - DEFVARS_mutexProtection; + wtp_t *pThis; - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - pThis->bInactivityGuard = bNewState; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + BEGINfunc + ISOBJ_TYPE_assert(pWti, wti); + pThis = pWti->pWtp; + ISOBJ_TYPE_assert(pThis, wtp); - RETiRet; + /* the order of the next two statements is important! */ + wtiSetState(pWti, WRKTHRD_STOPPED); + ATOMIC_DEC(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd); + + DBGPRINTF("%s: Worker thread %lx, terminated, um workers now %d\n", + wtpGetDbgHdr(pThis), (unsigned long) pWti, + ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd)); + + ENDfunc } -/* cancellation cleanup handler for executing worker - * decrements the worker counter - * rgerhards, 2008-01-20 +/* cancellation cleanup handler for executing worker decrements the worker counter. + * rgerhards, 2009-07-20 */ -void +static void wtpWrkrExecCancelCleanup(void *arg) { - wtp_t *pThis = (wtp_t*) arg; + wti_t *pWti = (wti_t*) arg; + wtp_t *pThis; BEGINfunc + ISOBJ_TYPE_assert(pWti, wti); + pThis = pWti->pWtp; ISOBJ_TYPE_assert(pThis, wtp); - pThis->iCurNumWrkThrd--; - wtpSignalWrkrTermination(pThis); + DBGPRINTF("%s: Worker thread %lx requested to be cancelled.\n", + wtpGetDbgHdr(pThis), (unsigned long) pWti); + + wtpWrkrExecCleanup(pWti); - dbgprintf("%s: thread CANCELED with %d workers running.\n", wtpGetDbgHdr(pThis), pThis->iCurNumWrkThrd); ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ } @@ -437,8 +346,6 @@ wtpWrkrExecCancelCleanup(void *arg) static void * wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in wtp! */ { - DEFiRet; - DEFVARS_mutexProtection; wti_t *pWti = (wti_t*) arg; wtp_t *pThis; sigset_t sigSet; @@ -447,13 +354,20 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in uchar thrdName[32] = "rs:"; # endif + BEGINfunc ISOBJ_TYPE_assert(pWti, wti); pThis = pWti->pWtp; ISOBJ_TYPE_assert(pThis, wtp); + /* block all signals */ sigfillset(&sigSet); pthread_sigmask(SIG_BLOCK, &sigSet, NULL); + /* but ignore SIGTTN, which we (ab)use to signal the thread to shutdown -- rgerhards, 2009-07-20 */ + sigemptyset(&sigSet); + sigaddset(&sigSet, SIGTTIN); + pthread_sigmask(SIG_UNBLOCK, &sigSet, NULL); + # if HAVE_PRCTL && defined PR_SET_NAME /* set thread name - we ignore if the call fails, has no harsh consequences... */ pszDbgHdr = wtpGetDbgHdr(pThis); @@ -463,41 +377,17 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in } # endif - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - - /* do some late initialization */ - - pthread_cleanup_push(wtpWrkrExecCancelCleanup, pThis); - - /* finally change to RUNNING state. We need to check if we actually should still run, - * because someone may have requested us to shut down even before we got a chance to do - * our init. That would be a bad race... -- rgerhards, 2008-01-16 - */ - wtiSetState(pWti, eWRKTHRD_RUNNING, 0, MUTEX_ALREADY_LOCKED); /* we are running now! */ - - do { - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - iRet = wtiWorker(pWti); /* just to make sure: this is NOT protected by the mutex! */ - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - } while(pThis->iCurNumWrkThrd == 1 && pThis->bInactivityGuard == 1); - /* inactivity guard prevents shutdown of all workers while one should be running due to race - * condition. It can lead to one more worker running than desired, but that is acceptable. After - * all, that worker will shutdown itself due to inactivity timeout. If, however, none were running - * when one was required, processing could come to a halt. -- rgerhards, 2008-01-21 - */ - + pthread_cleanup_push(wtpWrkrExecCancelCleanup, pWti); + wtiWorker(pWti); pthread_cleanup_pop(0); - pThis->iCurNumWrkThrd--; - wtpSignalWrkrTermination(pThis); - - dbgprintf("%s: Worker thread %lx, terminated, num workers now %d\n", - wtpGetDbgHdr(pThis), (unsigned long) pWti, pThis->iCurNumWrkThrd); - - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + wtpWrkrExecCleanup(pWti); ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ pthread_exit(0); } #pragma GCC diagnostic warning "-Wempty-body" @@ -505,27 +395,20 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in /* start a new worker */ static rsRetVal -wtpStartWrkr(wtp_t *pThis, int bLockMutex) +wtpStartWrkr(wtp_t *pThis) { - DEFiRet; - DEFVARS_mutexProtection; wti_t *pWti; int i; int iState; + DEFiRet; ISOBJ_TYPE_assert(pThis, wtp); - wtpProcessThrdChanges(pThis); // TODO: Performance: this causes a lot of FUTEX calls + d_pthread_mutex_lock(&pThis->mutWtp); - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - - pThis->iCurNumWrkThrd++; - - /* find free spot in thread table. If we find at least one worker that is in initialization, - * we do NOT start a new one. Let's give the other one a chance, first. - */ + /* find free spot in thread table. */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { - if(wtiGetState(pThis->pWrkr[i], LOCK_MUTEX) == eWRKTHRD_STOPPED) { + if(wtiGetState(pThis->pWrkr[i]) == WRKTHRD_STOPPED) { break; } } @@ -533,17 +416,21 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) if(i == pThis->iNumWorkerThreads) ABORT_FINALIZE(RS_RET_NO_MORE_THREADS); + if(i == 0 || pThis->toWrkShutdown == -1) { + wtiSetAlwaysRunning(pThis->pWrkr[i]); + } + pWti = pThis->pWrkr[i]; - wtiSetState(pWti, eWRKTHRD_RUN_CREATED, 0, LOCK_MUTEX); - iState = pthread_create(&(pWti->thrdID), NULL, wtpWorker, (void*) pWti); - dbgprintf("%s: started with state %d, num workers now %d\n", - wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd); + wtiSetState(pWti, WRKTHRD_RUNNING); + iState = pthread_create(&(pWti->thrdID), &pThis->attrThrd, wtpWorker, (void*) pWti); + ATOMIC_INC(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd); /* we got one more! */ - /* indicate we just started a worker and would like to see it running */ - wtpSetInactivityGuard(pThis, 1, MUTEX_ALREADY_LOCKED); + DBGPRINTF("%s: started with state %d, num workers now %d\n", + wtpGetDbgHdr(pThis), iState, + ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd)); finalize_it: - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); + d_pthread_mutex_unlock(&pThis->mutWtp); RETiRet; } @@ -560,7 +447,6 @@ rsRetVal wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) { DEFiRet; - DEFVARS_mutexProtection; int nMissing; /* number workers missing to run */ int i; @@ -569,29 +455,24 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) if(nMaxWrkr == 0) FINALIZE; - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); - if(nMaxWrkr > pThis->iNumWorkerThreads) /* limit to configured maximum */ nMaxWrkr = pThis->iNumWorkerThreads; - nMissing = nMaxWrkr - pThis->iCurNumWrkThrd; + nMissing = nMaxWrkr - ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd); if(nMissing > 0) { - dbgprintf("%s: high activity - starting %d additional worker thread(s).\n", wtpGetDbgHdr(pThis), nMissing); + DBGPRINTF("%s: high activity - starting %d additional worker thread(s).\n", + wtpGetDbgHdr(pThis), nMissing); /* start the rqtd nbr of workers */ for(i = 0 ; i < nMissing ; ++i) { - CHKiRet(wtpStartWrkr(pThis, MUTEX_ALREADY_LOCKED)); - } - } else { - if(nMaxWrkr > 0) { - dbgprintf("wtpAdviseMaxWorkers signals busy\n"); - wtpWakeupWrkr(pThis); + CHKiRet(wtpStartWrkr(pThis)); } + } else { + pthread_cond_signal(pThis->pcondBusy); } finalize_it: - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); RETiRet; } @@ -605,35 +486,9 @@ DEFpropSetMethPTR(wtp, pmutUsr, pthread_mutex_t) DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t) DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)) -DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)) -DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) -DEFpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*, void*)) -DEFpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)) - - -/* return the current number of worker threads. - * TODO: atomic operation would bring a nice performance - * enhancemcent - * rgerhards, 2008-01-27 - */ -int -wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex) -{ - DEFVARS_mutexProtection; - int iNumWrkr; - - BEGINfunc - ISOBJ_TYPE_assert(pThis, wtp); - - BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - iNumWrkr = pThis->iCurNumWrkThrd; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); - - ENDfunc - return iNumWrkr; -} +DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)) +DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)) +DEFpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)) /* set the debug header message @@ -657,7 +512,7 @@ wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg) pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ @@ -687,6 +542,5 @@ BEGINObjClassInit(wtp, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wtp) -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/wtp.h b/runtime/wtp.h index 640c3320..7e6b4394 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -28,18 +28,9 @@ #include "obj.h" #include "atomic.h" -/* commands and states for worker threads. */ -typedef enum { - eWRKTHRD_STOPPED = 0, /* worker thread is not running (either actually never ran or was shut down) */ - eWRKTHRD_TERMINATING = 1,/* worker thread has shut down, but some finalzing is still needed */ - /* ALL active states MUST be numerically higher than eWRKTHRD_TERMINATED and NONE must be lower! */ - eWRKTHRD_RUN_CREATED = 2,/* worker thread has been created, but not yet begun initialization (prob. not yet scheduled) */ - eWRKTHRD_RUN_INIT = 3, /* worker thread is initializing, but not yet fully running */ - eWRKTHRD_RUNNING = 4, /* worker thread is up and running and shall continue to do so */ - eWRKTHRD_SHUTDOWN = 5, /* worker thread is running but shall terminate when wtp is empty */ - eWRKTHRD_SHUTDOWN_IMMEDIATE = 6/* worker thread is running but shall terminate even if wtp is full */ - /* SHUTDOWN_IMMEDIATE MUST alsways be the numerically highest state! */ -} qWrkCmd_t; +/* states for worker threads. */ +#define WRKTHRD_STOPPED FALSE +#define WRKTHRD_RUNNING TRUE /* possible states of a worker thread pool */ @@ -51,37 +42,33 @@ typedef enum { /* the worker thread pool (wtp) object */ -typedef struct wtp_s { +struct wtp_s { BEGINobjInstance; wtpState_t wtpState; int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ struct wti_s **pWrkr;/* array with control structure for the worker thread(s) associated with this wtp */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ - bool bInactivityGuard;/* prevents inactivity due to race condition */ rsRetVal (*pConsumer)(void *); /* user-supplied consumer function for dewtpd messages */ /* synchronization variables */ - pthread_mutex_t mutThrdShutdwn; /* mutex to guard thread shutdown processing */ - pthread_mutex_t mut; /* mutex for the wtp's thread management */ + pthread_mutex_t mutWtp; /* mutex for the wtp's thread management */ pthread_cond_t condThrdTrm;/* signalled when threads terminate */ - int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* user objects */ - void *pUsr; /* pointer to user object */ + void *pUsr; /* pointer to user object (in this case, the queue the wtp belongs to) */ + pthread_attr_t attrThrd;/* attribute for new threads (created just once and cached here) */ pthread_mutex_t *pmutUsr; pthread_cond_t *pcondBusy; /* condition the user will signal "busy again, keep runing" on (awakes worker) */ rsRetVal (*pfChkStopWrkr)(void *pUsr, int); + rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */ + rsRetVal (*pfObjProcessed)(void *pUsr, wti_t *pWti); /* indicate user object is processed */ rsRetVal (*pfRateLimiter)(void *pUsr); - rsRetVal (*pfIsIdle)(void *pUsr, int); - rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int); - rsRetVal (*pfOnIdle)(void *pUsr, int); - rsRetVal (*pfOnWorkerCancel)(void *pUsr, void*pWti); - rsRetVal (*pfOnWorkerStartup)(void *pUsr); - rsRetVal (*pfOnWorkerShutdown)(void *pUsr); + rsRetVal (*pfDoWork)(void *pUsr, void *pWti); /* end user objects */ uchar *pszDbgHdr; /* header string for debug messages */ - DEF_ATOMIC_HELPER_MUT(mutThrdStateChanged); -} wtp_t; + DEF_ATOMIC_HELPER_MUT(mutCurNumWrkThrd); + DEF_ATOMIC_HELPER_MUT(mutWtpState); +}; /* some symbolic constants for easier reference */ @@ -92,26 +79,18 @@ rsRetVal wtpConstructFinalize(wtp_t *pThis); rsRetVal wtpDestruct(wtp_t **ppThis); rsRetVal wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr); rsRetVal wtpProcessThrdChanges(wtp_t *pThis); -rsRetVal wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex); -rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex); +rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex); rsRetVal wtpSetState(wtp_t *pThis, wtpState_t iNewState); -rsRetVal wtpWakeupWrkr(wtp_t *pThis); rsRetVal wtpWakeupAllWrkr(wtp_t *pThis); rsRetVal wtpCancelAll(wtp_t *pThis); rsRetVal wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg); -rsRetVal wtpSignalWrkrTermination(wtp_t *pWtp); rsRetVal wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout); -int wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex); -void wtpSetThrdStateChanged(wtp_t *pThis, int val); PROTOTYPEObjClassInit(wtp); PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*,void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)); +PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)); +PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)); +PROTOTYPEpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)); PROTOTYPEpropSetMeth(wtp, toWrkShutdown, long); PROTOTYPEpropSetMeth(wtp, wtpState, wtpState_t); PROTOTYPEpropSetMeth(wtp, iMaxWorkerThreads, int); diff --git a/shave-libtool.in b/shave-libtool.in deleted file mode 100644 index 54ebd690..00000000 --- a/shave-libtool.in +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2009, Damien Lespiau <damien.lespiau@gmail.com> -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -# we need sed -SED=@SED@ -if test -z "$SED" ; then -SED=sed -fi - -lt_unmangle () -{ - last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` -} - -# the real libtool to use -LIBTOOL="$1" -shift - -# if 1, don't print anything, the underlaying wrapper will do it -pass_though=0 - -# scan the arguments, keep the right ones for libtool, and discover the mode -preserved_args= - -# have we seen the --tag option of libtool in the command line ? -tag_seen=0 - -while test "$#" -gt 0; do - opt="$1" - shift - - case $opt in - --mode=*) - mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` - preserved_args="$preserved_args $opt" - ;; - -o) - lt_output="$1" - preserved_args="$preserved_args $opt" - ;; - --tag=*) - tag_seen=1 - preserved_args="$preserved_args $opt" - ;; - *) - preserved_args="$preserved_args $opt" - ;; - esac -done - -case "$mode" in -compile) - # shave will be called and print the actual CC/CXX/LINK line - preserved_args="$preserved_args --shave-mode=$mode" - pass_though=1 - ;; -link) - preserved_args="$preserved_args --shave-mode=$mode" - Q=" LINK " - ;; -*) - # let's u - # echo "*** libtool: Unimplemented mode: $mode, fill a bug report" - ;; -esac - -lt_unmangle "$lt_output" -output=$last_result - -# automake does not add a --tag switch to its libtool invocation when -# assembling a .s file and rely on libtool to infer the right action based -# on the compiler name. As shave is using CC to hook a wrapper, libtool gets -# confused. Let's detect these cases and add a --tag=CC option. -tag="" -if test $tag_seen -eq 0 -a x"$mode" = xcompile; then - tag="--tag=CC" -fi - -if test -z $V; then - if test $pass_though -eq 0; then - echo "$Q$output" - fi - $LIBTOOL --silent $tag $preserved_args -else - echo $LIBTOOL $tag $preserved_args - $LIBTOOL $tag $preserved_args -fi diff --git a/shave.in b/shave.in deleted file mode 100644 index afed42e1..00000000 --- a/shave.in +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2009, Damien Lespiau <damien.lespiau@gmail.com> -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -# we need sed -SED=@SED@ -if test -z "$SED" ; then -SED=sed -fi - -lt_unmangle () -{ - last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` -} - -# the tool to wrap (cc, cxx, ar, ranlib, ..) -tool="$1" -shift - -# the reel tool (to call) -REEL_TOOL="$1" -shift - -pass_through=0 -preserved_args= -while test "$#" -gt 0; do - opt="$1" - shift - - case $opt in - --shave-mode=*) - mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` - ;; - -o) - lt_output="$1" - preserved_args="$preserved_args $opt" - ;; - *) - preserved_args="$preserved_args $opt" - ;; - esac -done - -# mode=link is handled in the libtool wrapper -case "$mode,$tool" in -link,*) - pass_through=1 - ;; -*,cxx) - Q=" CXX " - ;; -*,cc) - Q=" CC " - ;; -*,fc) - Q=" FC " - ;; -*,f77) - Q=" F77 " - ;; -*,objc) - Q=" OBJC " - ;; -*,*) - # should not happen - Q=" CC " - ;; -esac - -lt_unmangle "$lt_output" -output=$last_result - -if test -z $V; then - if test $pass_through -eq 0; then - echo "$Q$output" - fi - $REEL_TOOL $preserved_args -else - echo $REEL_TOOL $preserved_args - $REEL_TOOL $preserved_args -fi @@ -169,7 +169,7 @@ TCPSendBldFrame(tcpclt_t *pThis, char **pmsg, size_t *plen, int *pbMustBeFreed) * I have added this comment so that the logic is not accidently * changed again. rgerhards, 2005-10-25 */ - if((buf = malloc((len + 2) * sizeof(char))) == NULL) { + if((buf = MALLOC((len + 2) * sizeof(char))) == NULL) { /* extreme mem shortage, try to solve * as good as we can. No point in calling * any alarms, they might as well run out @@ -217,7 +217,7 @@ TCPSendBldFrame(tcpclt_t *pThis, char **pmsg, size_t *plen, int *pbMustBeFreed) /* IETF20061218 iLenBuf = snprintf(szLenBuf, sizeof(szLenBuf)/sizeof(char), "%d ", len + iLenBuf);*/ - if((buf = malloc((len + iLenBuf) * sizeof(char))) == NULL) { + if((buf = MALLOC((len + iLenBuf) * sizeof(char))) == NULL) { /* we are out of memory. This is an extreme situation. We do not * call any alarm handlers because they most likely run out of mem, * too. We are brave enough to call debug output, though. Other than @@ -324,7 +324,7 @@ Send(tcpclt_t *pThis, void *pData, char *msg, size_t len) * happens is that we lose our message recovery buffer - anything else would * be worse, so don't try anything ;) -- rgerhards, 2008-03-12 */ - if((pThis->prevMsg = malloc(len)) != NULL) { + if((pThis->prevMsg = MALLOC(len)) != NULL) { memcpy(pThis->prevMsg, msg, len); pThis->lenPrevMsg = len; } diff --git a/tcps_sess.c b/tcps_sess.c index ea9032b3..99af0cb8 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -47,6 +47,7 @@ #include "msg.h" #include "datetime.h" #include "prop.h" +#include "debug.h" /* static data */ @@ -70,7 +71,7 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ /* now allocate the message reception buffer */ - CHKmalloc(pThis->pMsg = (uchar*) malloc(sizeof(uchar) * iMaxLine + 1)); + CHKmalloc(pThis->pMsg = (uchar*) MALLOC(sizeof(uchar) * iMaxLine + 1)); finalize_it: ENDobjConstruct(tcps_sess) @@ -254,7 +255,6 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG MsgSetInputName(pMsg, pThis->pLstnInfo->pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; - pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pThis->fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); @@ -410,7 +410,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt */ } - if(( (c == '\n') + if(( ((c == '\n') && !pThis->pSrv->bDisableLFDelim) || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); @@ -15,12 +15,9 @@ * callbacks before the code is run. The tcpsrv then calls back * into the specific input modules at the appropriate time. * - * NOTE: read comments in module-template.h to understand how this file - * works! - * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -68,6 +65,7 @@ #include "netstrms.h" #include "netstrm.h" #include "nssel.h" +#include "nspoll.h" #include "errmsg.h" #include "ruleset.h" #include "unicode-helper.h" @@ -89,6 +87,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) +DEFobjCurrIf(nspoll) DEFobjCurrIf(prop) @@ -104,7 +103,7 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) ISOBJ_TYPE_assert(pThis, tcpsrv); /* create entry */ - CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); + CHKmalloc(pEntry = MALLOC(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; pEntry->pRuleset = pThis->pRuleset; @@ -165,9 +164,9 @@ TCPSessTblInit(tcpsrv_t *pThis) ISOBJ_TYPE_assert(pThis, tcpsrv); assert(pThis->pSessions == NULL); - dbgprintf("Allocating buffer for %d TCP sessions.\n", pThis->iSessMax); + DBGPRINTF("Allocating buffer for %d TCP sessions.\n", pThis->iSessMax); if((pThis->pSessions = (tcps_sess_t **) calloc(pThis->iSessMax, sizeof(tcps_sess_t *))) == NULL) { - dbgprintf("Error: TCPSessInit() could not alloc memory for TCP session table.\n"); + DBGPRINTF("Error: TCPSessInit() could not alloc memory for TCP session table.\n"); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -238,11 +237,13 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) if(pThis->pSessions != NULL) { /* close all TCP connections! */ - i = TCPSessGetNxtSess(pThis, -1); - while(i != -1) { - tcps_sess.Destruct(&pThis->pSessions[i]); - /* now get next... */ - i = TCPSessGetNxtSess(pThis, i); + if(!pThis->bUsingEPoll) { + i = TCPSessGetNxtSess(pThis, -1); + while(i != -1) { + tcps_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = TCPSessGetNxtSess(pThis, i); + } } /* we are done with the session table - so get rid of it... */ @@ -412,7 +413,7 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, * rgerhards, 2005-09-26 */ if(!pThis->pIsPermittedHost((struct sockaddr*) addr, (char*) fromHostFQDN, pThis->pUsr, pSess->pUsr)) { - dbgprintf("%s is not an allowed sender\n", fromHostFQDN); + DBGPRINTF("%s is not an allowed sender\n", fromHostFQDN); if(glbl.GetOption_DisallowWarning()) { errno = 0; errmsg.LogError(0, RS_RET_HOST_NOT_PERMITTED, "TCP message from disallowed sender %s discarded", fromHostFQDN); @@ -438,7 +439,8 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, } *ppSess = pSess; - pThis->pSessions[iSess] = pSess; + if(!pThis->bUsingEPoll) + pThis->pSessions[iSess] = pSess; pSess = NULL; /* this is now also handed over */ finalize_it: @@ -465,10 +467,85 @@ RunCancelCleanup(void *arg) } -/* This function is called to gather input. */ -#pragma GCC diagnostic ignored "-Wempty-body" +/* helper to close a session. Takes status of poll vs. select into consideration. + * rgerhards, 2009-11-25 + */ +static inline rsRetVal +closeSess(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) { + DEFiRet; + if(pPoll != NULL) { + CHKiRet(nspoll.Ctl(pPoll, (*ppSess)->pStrm, 0, *ppSess, NSDPOLL_IN, NSDPOLL_DEL)); + } + pThis->pOnRegularClose(*ppSess); + tcps_sess.Destruct(ppSess); +finalize_it: + RETiRet; +} + + +/* process a receive request on one of the streams + * If pPoll is non-NULL, we have a netstream in epoll mode, which means we need + * to remove any descriptor we close from the epoll set. + * rgerhards, 2009-07-020 + */ static rsRetVal -Run(tcpsrv_t *pThis) +doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) +{ + char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ + ssize_t iRcvd; + rsRetVal localRet; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + DBGPRINTF("netstream %p with new data\n", (*ppSess)->pStrm); + /* Receive message */ + iRet = pThis->pRcvData(*ppSess, buf, sizeof(buf), &iRcvd); + switch(iRet) { + case RS_RET_CLOSED: + if(pThis->bEmitMsgOnClose) { + uchar *pszPeer; + int lenPeer; + errno = 0; + prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); + errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "Netstream session %p closed by remote peer %s.\n", + (*ppSess)->pStrm, pszPeer); + } + CHKiRet(closeSess(pThis, ppSess, pPoll)); + break; + case RS_RET_RETRY: + /* we simply ignore retry - this is not an error, but we also have not received anything */ + break; + case RS_RET_OK: + /* valid data received, process it! */ + if((localRet = tcps_sess.DataRcvd(*ppSess, buf, iRcvd)) != RS_RET_OK) { + /* in this case, something went awfully wrong. + * We are instructed to terminate the session. + */ + errmsg.LogError(0, localRet, "Tearing down TCP Session - see " + "previous messages for reason(s)\n"); + CHKiRet(closeSess(pThis, ppSess, pPoll)); + } + break; + default: + errno = 0; + errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", + (*ppSess)->pStrm); + CHKiRet(closeSess(pThis, ppSess, pPoll)); + break; + } + +finalize_it: + RETiRet; +} + + +/* This function is called to gather input. + * This variant here is only used if we need to work with a netstream driver + * that does not support epoll(). + */ +#pragma GCC diagnostic ignored "-Wempty-body" +static inline rsRetVal +RunSelect(tcpsrv_t *pThis) { DEFiRet; int nfds; @@ -476,14 +553,13 @@ Run(tcpsrv_t *pThis) int iTCPSess; int bIsReady; tcps_sess_t *pNewSess; - nssel_t *pSel; - ssize_t iRcvd; + nssel_t *pSel = NULL; ISOBJ_TYPE_assert(pThis, tcpsrv); /* this is an endless loop - it is terminated by the framework canelling * this thread. Thus, we also need to instantiate a cancel cleanup handler - * to prevent us from leaking anything. -- rgerharsd, 20080-04-24 + * to prevent us from leaking anything. -- rgerhards, 20080-04-24 */ pthread_cleanup_push(RunCancelCleanup, (void*) &pSel); while(1) { @@ -507,11 +583,15 @@ Run(tcpsrv_t *pThis) /* wait for io to become ready */ CHKiRet(nssel.Wait(pSel, &nfds)); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + if(glbl.GetGlobalInputTermState() == 1) + ABORT_FINALIZE(RS_RET_FORCE_TERM); CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); + DBGPRINTF("New connect on NSD %p.\n", pThis->ppLstn[i]); SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); --nfds; /* indicate we have processed one */ } @@ -520,49 +600,11 @@ Run(tcpsrv_t *pThis) /* now check the sessions */ iTCPSess = TCPSessGetNxtSess(pThis, -1); while(nfds && iTCPSess != -1) { + if(glbl.GetGlobalInputTermState() == 1) + ABORT_FINALIZE(RS_RET_FORCE_TERM); CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ - dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); - - /* Receive message */ - iRet = pThis->pRcvData(pThis->pSessions[iTCPSess], buf, sizeof(buf), &iRcvd); - switch(iRet) { - case RS_RET_CLOSED: - if(pThis->bEmitMsgOnClose) { - uchar *pszPeer; - int lenPeer; - errno = 0; - prop.GetString(pThis->pSessions[iTCPSess]->fromHostIP, &pszPeer, &lenPeer); - errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "Netstream session %p closed by remote peer %s.\n", - pThis->pSessions[iTCPSess]->pStrm, pszPeer); - } - pThis->pOnRegularClose(pThis->pSessions[iTCPSess]); - tcps_sess.Destruct(&pThis->pSessions[iTCPSess]); - break; - case RS_RET_RETRY: - /* we simply ignore retry - this is not an error, but we also have not received anything */ - break; - case RS_RET_OK: - /* valid data received, process it! */ - if(tcps_sess.DataRcvd(pThis->pSessions[iTCPSess], buf, iRcvd) != RS_RET_OK) { - /* in this case, something went awfully wrong. - * We are instructed to terminate the session. - */ - errmsg.LogError(0, NO_ERRCODE, "Tearing down TCP Session %d - see " - "previous messages for reason(s)\n", iTCPSess); - pThis->pOnErrClose(pThis->pSessions[iTCPSess]); - tcps_sess.Destruct(&pThis->pSessions[iTCPSess]); - } - break; - default: - errno = 0; - errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", - pThis->pSessions[iTCPSess]->pStrm); - pThis->pOnErrClose(pThis->pSessions[iTCPSess]); - tcps_sess.Destruct(&pThis->pSessions[iTCPSess]); - break; - } + doReceive(pThis, &pThis->pSessions[iTCPSess], NULL); --nfds; /* indicate we have processed one */ } iTCPSess = TCPSessGetNxtSess(pThis, iTCPSess); @@ -578,18 +620,100 @@ finalize_it: /* this is a very special case - this time only we do not exit the } /* note that this point is usually not reached */ - pthread_cleanup_pop(0); /* remove cleanup handler */ + pthread_cleanup_pop(1); /* remove cleanup handler */ RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" +/* This function is called to gather input. It tries doing that via the epoll() + * interface. If the driver does not support that, it falls back to calling its + * select() equivalent. + * rgerhards, 2009-11-18 + */ +static rsRetVal +Run(tcpsrv_t *pThis) +{ + DEFiRet; + int i; + tcps_sess_t *pNewSess; + nspoll_t *pPoll = NULL; + void *pUsr; + rsRetVal localRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerhards, 20080-04-24 + */ + if((localRet = nspoll.Construct(&pPoll)) == RS_RET_OK) { + // TODO: set driver + localRet = nspoll.ConstructFinalize(pPoll); + } + if(localRet != RS_RET_OK) { + /* fall back to select */ + dbgprintf("tcpsrv could not use epoll() interface, iRet=%d, using select()\n", localRet); + iRet = RunSelect(pThis); + FINALIZE; + } + + dbgprintf("tcpsrv uses epoll() interface, nsdpol driver found\n"); + + /* flag that we are in epoll mode */ + pThis->bUsingEPoll = TRUE; + + /* Add the TCP listen sockets to the list of sockets to monitor */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + dbgprintf("Trying to add listener %d, pUsr=%p\n", i, pThis->ppLstn); + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_ADD)); + dbgprintf("Added listener %d\n", i); + } + + while(1) { + localRet = nspoll.Wait(pPoll, -1, &i, &pUsr); + if(glbl.GetGlobalInputTermState() == 1) + break; /* terminate input! */ + + /* check if we need to ignore the i/o ready state. We do this if we got an invalid + * return state. Validly, this can happen for RS_RET_EINTR, for other cases it may + * not be the right thing, but what is the right thing is really hard at this point... + */ + if(localRet != RS_RET_OK) + continue; + + dbgprintf("poll returned with i %d, pUsr %p\n", i, pUsr); + + if(pUsr == pThis->ppLstn) { + DBGPRINTF("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + CHKiRet(nspoll.Ctl(pPoll, pNewSess->pStrm, 0, pNewSess, NSDPOLL_IN, NSDPOLL_ADD)); + DBGPRINTF("New session created with NSD %p.\n", pNewSess); + } else { + pNewSess = (tcps_sess_t*) pUsr; + doReceive(pThis, &pNewSess, pPoll); + } + } + + /* remove the tcp listen sockets from the epoll set */ + for(i = 0 ; i < pThis->iLstnCurr ; ++i) { + CHKiRet(nspoll.Ctl(pPoll, pThis->ppLstn[i], i, pThis->ppLstn, NSDPOLL_IN, NSDPOLL_DEL)); + } + +finalize_it: + if(pPoll != NULL) + nspoll.Destruct(&pPoll); + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ pThis->iSessMax = TCPSESS_MAX_DEFAULT; pThis->iLstnMax = TCPLSTN_MAX_DEFAULT; pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + pThis->bDisableLFDelim = 0; pThis->OnMsgReceive = NULL; ENDobjConstruct(tcpsrv) @@ -746,6 +870,18 @@ SetOnMsgReceive(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, } +/* set enable/disable standard LF frame delimiter (use with care!) + * -- rgerhards, 2010-01-03 + */ +static rsRetVal +SetbDisableLFDelim(tcpsrv_t *pThis, int bVal) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->bDisableLFDelim = bVal; + RETiRet; +} + /* Set additional framing to use (if any) -- rgerhards, 2008-12-10 */ static rsRetVal @@ -887,7 +1023,6 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->ConstructFinalize = tcpsrvConstructFinalize; pIf->Destruct = tcpsrvDestruct; - //pIf->SessAccept = SessAccept; pIf->configureTCPListen = configureTCPListen; pIf->create_tcp_socket = create_tcp_socket; pIf->Run = Run; @@ -895,6 +1030,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetUsrP = SetUsrP; pIf->SetInputName = SetInputName; pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; + pIf->SetbDisableLFDelim = SetbDisableLFDelim; pIf->SetSessMax = SetSessMax; pIf->SetLstnMax = SetLstnMax; pIf->SetDrvrMode = SetDrvrMode; @@ -948,6 +1084,7 @@ BEGINObjClassInit(tcpsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(nspoll, DONT_LOAD_LIB)); CHKiRet(objUse(tcps_sess, DONT_LOAD_LIB)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); @@ -54,7 +54,8 @@ struct tcpsrv_s { uchar *pszInputName; /**< value to be used as input name */ ruleset_t *pRuleset; /**< ruleset to bind to */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ - bool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ + sbool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ + sbool bUsingEPoll; /**< are we in epoll mode (means we do not need to keep track of sessions!) */ int iLstnCurr; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ @@ -63,6 +64,7 @@ struct tcpsrv_s { tcpLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ int addtlFrameDelim; /**< additional frame delimiter for plain TCP syslog framing (e.g. to handle NetScreen) */ + int bDisableLFDelim; /**< if 1, standard LF frame delimiter is disabled (*very dangerous*) */ tcps_sess_t **pSessions;/**< array of all of our sessions */ void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ /* callbacks */ @@ -114,11 +116,13 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ /* added v6 */ rsRetVal (*SetOnMsgReceive)(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); /* 2009-05-24 */ rsRetVal (*SetRuleset)(tcpsrv_t *pThis, ruleset_t*); /* 2009-06-12 */ - /* added v7 */ + /* added v7 (accidently named v8!) */ rsRetVal (*SetLstnMax)(tcpsrv_t *pThis, int iMaxLstn); /* 2009-08-17 */ rsRetVal (*SetNotificationOnRemoteClose)(tcpsrv_t *pThis, int bNewVal); /* 2009-10-01 */ + /* added v9 -- rgerhards, 2010-03-01 */ + rsRetVal (*SetbDisableLFDelim)(tcpsrv_t*, int); ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 8 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 @@ -36,11 +36,14 @@ #include "dirty.h" #include "obj.h" #include "errmsg.h" +#include "strgen.h" +#include "unicode-helper.h" /* static data */ DEFobjCurrIf(obj) DEFobjCurrIf(errmsg) DEFobjCurrIf(regexp) +DEFobjCurrIf(strgen) static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */ static struct template *tplRoot = NULL; /* the root of the template list */ @@ -49,9 +52,10 @@ static struct template *tplLastStatic = NULL; /* last static element of the temp -/* helper to tplToString, extends buffer */ +/* helper to tplToString and strgen's, extends buffer */ #define ALLOC_INC 128 -static inline rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize) +rsRetVal +ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize) { uchar *pNewBuf; size_t iNewSize; @@ -91,6 +95,11 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * assert(ppBuf != NULL); assert(pLenBuf != NULL); + if(pTpl->pStrgen != NULL) { + CHKiRet(pTpl->pStrgen(pMsg, ppBuf, pLenBuf)); + FINALIZE; + } + /* loop through the template. We obtain one value * and copy it over to our dynamic string buffer. Then, we * free the obtained value (if requested). We continue this @@ -117,10 +126,11 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); } /* got source, now copy over */ - if(iBuf + iLenVal + 1 >= *pLenBuf) /* we reserve one char for the final \0! */ - CHKiRet(ExtendBuf(ppBuf, pLenBuf, iBuf + iLenVal + 1)); - if(iLenVal > 0) { /* may be zero depending on property */ + /* first, make sure buffer fits */ + if(iBuf + iLenVal >= *pLenBuf) /* we reserve one char for the final \0! */ + CHKiRet(ExtendBuf(ppBuf, pLenBuf, iBuf + iLenVal + 1)); + memcpy(*ppBuf + iBuf, pVal, iLenVal); iBuf += iLenVal; } @@ -741,7 +751,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) /* We get here ONLY if the regex end was found */ longitud = regex_end - p; /* Malloc for the regex string */ - regex_char = (unsigned char *) malloc(longitud + 1); + regex_char = (unsigned char *) MALLOC(longitud + 1); if(regex_char == NULL) { dbgprintf("Could not allocate memory for template parameter!\n"); pTpe->data.field.has_regex = 0; @@ -829,16 +839,45 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) } +/* Add a new entry for a template module. + * returns pointer to new object if it succeeds, NULL otherwise. + * rgerhards, 2010-05-31 + */ +static rsRetVal +tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) +{ + uchar *pSrc, *pDst; + uchar szMod[2048]; + strgen_t *pStrgen; + DEFiRet; + + pSrc = *ppRestOfConfLine; + pDst = szMod; + while(*pSrc && !isspace(*pSrc) && pDst < &(szMod[sizeof(szMod) - 1])) { + *pDst++ = *pSrc++; + } + *pDst = '\0'; + *ppRestOfConfLine = pSrc; + CHKiRet(strgen.FindStrgen(&pStrgen, szMod)); + pTpl->pStrgen = pStrgen->pModule->mod.sm.strgen; + dbgprintf("template bound to strgen '%s'\n", szMod); + +finalize_it: + RETiRet; +} + + /* Add a new template line * returns pointer to new object if it succeeds, NULL otherwise. */ -struct template *tplAddLine(char* pName, unsigned char** ppRestOfConfLine) +struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) { struct template *pTpl; unsigned char *p; int bDone; char optBuf[128]; /* buffer for options - should be more than enough... */ size_t i; + rsRetVal localRet; assert(pName != NULL); assert(ppRestOfConfLine != NULL); @@ -847,7 +886,7 @@ struct template *tplAddLine(char* pName, unsigned char** ppRestOfConfLine) return NULL; pTpl->iLenName = strlen(pName); - pTpl->pszName = (char*) malloc(sizeof(char) * (pTpl->iLenName + 1)); + pTpl->pszName = (char*) MALLOC(sizeof(char) * (pTpl->iLenName + 1)); if(pTpl->pszName == NULL) { dbgprintf("tplAddLine could not alloc memory for template name!"); pTpl->iLenName = 0; @@ -866,7 +905,25 @@ struct template *tplAddLine(char* pName, unsigned char** ppRestOfConfLine) while(isspace((int)*p))/* skip whitespace */ ++p; - if(*p != '"') { + switch(*p) { + case '"': /* just continue */ + break; + case '=': + *ppRestOfConfLine = p + 1; + localRet = tplAddTplMod(pTpl, ppRestOfConfLine); + if(localRet != RS_RET_OK) { + errmsg.LogError(0, localRet, "Template '%s': error %d defining template via strgen module", + pTpl->pszName, localRet); + /* we simply make the template defunct in this case by setting + * its name to a zero-string. We do not free it, as this would + * require additional code and causes only a very small memory + * consumption. Memory is freed, however, in normal operation + * and most importantly by HUPing syslogd. + */ + *pTpl->pszName = '\0'; + } + return NULL; + default: dbgprintf("Template '%s' invalid, does not start with '\"'!\n", pTpl->pszName); /* we simply make the template defunct in this case by setting * its name to a zero-string. We do not free it, as this would @@ -942,6 +999,7 @@ struct template *tplAddLine(char* pName, unsigned char** ppRestOfConfLine) } *ppRestOfConfLine = p; + return(pTpl); } @@ -1194,11 +1252,8 @@ rsRetVal templateInit() DEFiRet; CHKiRet(objGetObjInterface(&obj)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(strgen, CORE_COMPONENT)); finalize_it: RETiRet; } - -/* - * vi:set ai: - */ @@ -32,6 +32,7 @@ struct template { struct template *pNext; char *pszName; int iLenName; + rsRetVal (*pStrgen)(msg_t*, uchar**, size_t *); /* name of strgen to use (bound if non-NULL!) */ int tpenElements; /* number of elements in templateEntry list */ struct templateEntry *pEntryRoot; struct templateEntry *pEntryLast; @@ -121,6 +122,7 @@ void tplDeleteAll(void); void tplDeleteNew(void); void tplPrintList(void); void tplLastStaticInit(struct template *tpl); +rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); /* note: if a compiler warning for undefined type tells you to look at this * code line below, the actual cause is that you currently MUST include template.h * BEFORE msg.h, even if your code file does not actually need it. diff --git a/tests/Makefile.am b/tests/Makefile.am index ac0844e9..feb33e49 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -2,10 +2,15 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr TESTS = $(TESTRUNS) cfg.sh \ + arrayqueue.sh \ + linkedlistqueue.sh \ + da-mainmsg-q.sh \ validation-run.sh \ imtcp-multiport.sh \ + daqueue-persist.sh \ diskqueue.sh \ diskqueue-fsync.sh \ + rulesetmultiqueue.sh \ manytcp.sh \ rsf_getenv.sh \ manyptcp.sh \ @@ -14,6 +19,8 @@ TESTS = $(TESTRUNS) cfg.sh \ imtcp_conndrop.sh \ sndrcv.sh \ sndrcv_gzip.sh \ + sndrcv_udp.sh \ + sndrcv_udp_nonstdpt.sh \ asynwr_simple.sh \ asynwr_timeout.sh \ asynwr_small.sh \ @@ -33,8 +40,20 @@ TESTS = $(TESTRUNS) cfg.sh \ pipeaction.sh \ uxsock_simple.sh \ execonlyonce.sh \ + execonlywhenprevsuspended.sh \ + execonlywhenprevsuspended2.sh \ + execonlywhenprevsuspended3.sh \ + execonlywhenprevsuspended4.sh \ + pipe_noreader.sh \ + dircreate_dflt.sh \ + dircreate_off.sh \ queue-persist.sh +if ENABLE_OMUDPSPOOF +TESTS += sndrcv_omudpspoof.sh \ + sndrcv_omudpspoof_nonstdpt.sh +endif + if ENABLE_OMSTDOUT TESTS += omod-if-array.sh \ proprepltest.sh \ @@ -43,9 +62,18 @@ TESTS += omod-if-array.sh \ inputname.sh \ threadingmq.sh \ threadingmqaq.sh \ + discard.sh \ + badqi.sh \ + tabescape_dflt.sh \ + tabescape_off.sh \ fieldtest.sh endif +if ENABLE_OMRULESET +TESTS += omruleset.sh \ + omruleset-queue.sh +endif + if ENABLE_EXTENDED_TESTS TESTS += random.sh endif @@ -96,11 +124,17 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/master.tspgsql \ testsuites/subsecond.conf \ testsuites/master.subsecond \ + testsuites/parse_8bit_escape.conf \ + testsuites/8bit.parse_8bit_escape \ testsuites/parse1.conf \ testsuites/field1.conf \ testsuites/1.parse1 \ testsuites/2.parse1 \ testsuites/3.parse1 \ + testsuites/4.parse1 \ + testsuites/mark.parse1 \ + testsuites/8bit.parse1 \ + testsuites/empty.parse1 \ testsuites/snare.parse1 \ testsuites/oversizeTag-1.parse1 \ testsuites/weird.parse1 \ @@ -114,6 +148,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rfc5424-2.parse1 \ testsuites/rfc5424-3.parse1 \ testsuites/rfc5424-4.parse1 \ + testsuites/malformed1.parse1 \ + testsuites/reallife.parse1 \ + testsuites/parse2.conf \ + testsuites/reallife.parse2 \ testsuites/parse3.conf \ testsuites/reallife.parse3 \ testsuites/parse-nodate.conf \ @@ -136,6 +174,12 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rsf_getenv.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ + arrayqueue.sh \ + testsuites/arrayqueue.conf \ + linkedlistqueue.sh \ + testsuites/linkedlistqueue.conf \ + da-mainmsg-q.sh \ + testsuites/da-mainmsg-q.conf \ diskqueue-fsync.sh \ testsuites/diskqueue-fsync.conf \ imtcp-multiport.sh \ @@ -156,9 +200,13 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12515 \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ + discard.sh \ + testsuites/discard.conf \ diag.sh \ testsuites/diag-common.conf \ testsuites/diag-common2.conf \ + daqueue-persist.sh \ + daqueue-persist-drvr.sh \ queue-persist.sh \ queue-persist-drvr.sh \ testsuites/queue-persist.conf \ @@ -170,11 +218,25 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ sndrcv.sh \ testsuites/sndrcv_sender.conf \ testsuites/sndrcv_rcvr.conf \ + sndrcv_udp.sh \ + testsuites/sndrcv_udp_sender.conf \ + testsuites/sndrcv_udp_rcvr.conf \ + sndrcv_udp_nonstdpt.sh \ + testsuites/sndrcv_udp_nonstdpt_sender.conf \ + testsuites/sndrcv_udp_nonstdpt_rcvr.conf \ + sndrcv_omudpspoof.sh \ + testsuites/sndrcv_omudpspoof_sender.conf \ + testsuites/sndrcv_omudpspoof_rcvr.conf \ + sndrcv_omudpspoof_nonstdpt.sh \ + testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf \ + testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf \ sndrcv_gzip.sh \ testsuites/sndrcv_gzip_sender.conf \ testsuites/sndrcv_gzip_rcvr.conf \ pipeaction.sh \ testsuites/pipeaction.conf \ + pipe_noreader.sh \ + testsuites/pipe_noreader.conf \ uxsock_simple.sh \ testsuites/uxsock_simple.conf \ asynwr_simple.sh \ @@ -214,9 +276,36 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/master.rfctag \ testsuites/nolimittag.conf \ testsuites/master.nolimittag \ + rulesetmultiqueue.sh \ + testsuites/rulesetmultiqueue.conf \ + omruleset.sh \ + testsuites/omruleset.conf \ + omruleset-queue.sh \ + testsuites/omruleset-queue.conf \ + badqi.sh \ + testsuites/badqi.conf \ + bad_qi/dbq.qi \ execonlyonce.sh \ testsuites/execonlyonce.conf \ testsuites/execonlyonce.data \ + execonlywhenprevsuspended.sh \ + testsuites/execonlywhenprevsuspended.conf \ + execonlywhenprevsuspended2.sh \ + testsuites/execonlywhenprevsuspended2.conf \ + execonlywhenprevsuspended3.sh \ + testsuites/execonlywhenprevsuspended3.conf \ + execonlywhenprevsuspended4.sh \ + testsuites/execonlywhenprevsuspended4.conf \ + tabescape_dflt.sh \ + testsuites/tabescape_dflt.conf \ + testsuites/1.tabescape_dflt \ + tabescape_off.sh \ + testsuites/tabescape_off.conf \ + testsuites/1.tabescape_off \ + dircreate_dflt.sh \ + testsuites/dircreate_dflt.conf \ + dircreate_off.sh \ + testsuites/dircreate_off.conf \ cfg.sh uxsockrcvr_SOURCES = uxsockrcvr.c diff --git a/tests/arrayqueue.sh b/tests/arrayqueue.sh new file mode 100755 index 00000000..71e1cc21 --- /dev/null +++ b/tests/arrayqueue.sh @@ -0,0 +1,18 @@ +# Test for fixedArray queue mode +# added 2009-05-20 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[arrayqueue.sh\]: testing queue fixedArray queue mode +source $srcdir/diag.sh init +source $srcdir/diag.sh startup arrayqueue.conf + +# 40000 messages should be enough +source $srcdir/diag.sh injectmsg 0 40000 + +# terminate *now* (don't wait for queue to drain!) +kill `cat rsyslog.pid` + +# now wait until rsyslog.pid is gone (and the process finished) +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/bad_qi/dbq.qi b/tests/bad_qi/dbq.qi new file mode 100644 index 00000000..5f56e41a --- /dev/null +++ b/tests/bad_qi/dbq.qi @@ -0,0 +1,29 @@ +!OPB:1:queue:1: ++iQueueSize:2:2:84: ++iUngottenObjs:2:1:1: ++tVars.disk.sizeOnDisk:2:5:57906: ++tVars.disk.bytesRead:2:4:1010: +>End +. +<Obj:1:strm:1: ++iCurrFNum:2:1:1: ++pszFName:1:3:dbq: ++iMaxFiles:2:8:10000000: ++bDeleteOnClose:2:1:0: ++sType:2:1:1: ++tOperationsMode:2:1:2: ++tOpenMode:2:3:384: ++iCurrOffs:2:5:57906: +>End +. +<Obj:1:strm:1: ++iCurrFNum:2:1:1: ++pszFName:1:3:dbq: ++iMaxFiles:2:8:10000000: ++bDeleteOnClose:2:1:1: ++sType:2:1:1: ++tOperationsMode:2:1:1: ++tOpenMode:2:3:384: ++iCurrOffs:2:4:1010: +>End +. diff --git a/tests/badqi.sh b/tests/badqi.sh new file mode 100755 index 00000000..28f76229 --- /dev/null +++ b/tests/badqi.sh @@ -0,0 +1,16 @@ +# Test for a startup with a bad qi file. This tests simply tests +# if the rsyslog engine survives (we had segfaults in this situation +# in the past). +# added 2009-10-21 by RGerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =============================================================================== +echo \[badqi.sh\]: test startup with invalid .qi file +source $srcdir/diag.sh init +source $srcdir/diag.sh startup badqi.conf +# we just inject a handful of messages so that we have something to wait for... +source $srcdir/diag.sh tcpflood -m20 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # wait for process to terminate +source $srcdir/diag.sh seq-check 0 19 +source $srcdir/diag.sh exit diff --git a/tests/cfg.sh b/tests/cfg.sh index cb838354..f850c4d1 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -31,6 +31,7 @@ # # A copy of the GPL can be found in the file "COPYING" in this distribution. #set -x +echo \[cfg.sh\]: rm -f tmp echo "local directory" # diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh new file mode 100755 index 00000000..d9cc0d4d --- /dev/null +++ b/tests/da-mainmsg-q.sh @@ -0,0 +1,33 @@ +# Test for DA mode on the main message queue +# This test checks if DA mode operates correctly. To do so, +# it uses a small in-memory queue size, so that DA mode is initiated +# rather soon, and disk spooling used. There is some uncertainty (based +# on machine speeds), but in general the test should work rather well. +# We add a few messages after the initial run, just so that we can +# check everything recovers from DA mode correctly. +# added 2009-04-22 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo "[da-mainmsg-q.sh]: testing main message queue in DA mode (going to disk)" +source $srcdir/diag.sh init +source $srcdir/diag.sh startup da-mainmsg-q.conf + +# part1: send first 50 messages (in memory, only) +#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 50 +source $srcdir/diag.sh injectmsg 0 50 +source $srcdir/diag.sh wait-queueempty # let queue drain for this test case + +# part 2: send bunch of messages. This should trigger DA mode +#source $srcdir/diag.sh injectmsg 50 20000 +source $srcdir/diag.sh injectmsg 50 2000 +ls -l test-spool # for manual review + +# send another handful +source $srcdir/diag.sh injectmsg 2050 50 +#sleep 1 # we need this so that rsyslogd can receive all outstanding messages + +# clean up and check test result +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 2099 +source $srcdir/diag.sh exit diff --git a/tests/daqueue-persist-drvr.sh b/tests/daqueue-persist-drvr.sh new file mode 100755 index 00000000..7934eb2b --- /dev/null +++ b/tests/daqueue-persist-drvr.sh @@ -0,0 +1,36 @@ +# Test for queue data persisting at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo \[daqueue-persist-drvr.sh\]: testing memory daqueue persisting to disk, mode $1 +source $srcdir/diag.sh init + +#export RSYSLOG_DEBUG="debug nologfuncflow nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="log" + +# prepare config +echo \$MainMsgQueueType $1 > work-queuemode.conf +echo "*.* :omtesting:sleep 0 1000" > work-delay.conf + +# inject 10000 msgs, so that DO hit the high watermark +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh injectmsg 0 10000 +$srcdir/diag.sh shutdown-immediate +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh check-mainq-spool + +echo "Enter phase 2, rsyslogd restart" + +exit + +# restart engine and have rest processed +#remove delay +echo "#" > work-delay.conf +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99999 +source $srcdir/diag.sh exit diff --git a/tests/daqueue-persist.sh b/tests/daqueue-persist.sh new file mode 100755 index 00000000..feb2a347 --- /dev/null +++ b/tests/daqueue-persist.sh @@ -0,0 +1,12 @@ +# Test for queue data persisting at shutdown. We use the actual driver +# to carry out multiple tests with different queue modes +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo \[daqueue-persist.sh\]: test data persisting at shutdown +source $srcdir/daqueue-persist-drvr.sh LinkedList +source $srcdir/daqueue-persist-drvr.sh FixedArray +# the disk test should not fail, however, the config is extreme and using +# it more or less is a config error +source $srcdir/daqueue-persist-drvr.sh Disk +# we do not test Direct mode because this absolute can not work in direct mode +# (maybe we should do a fail-type of test?) diff --git a/tests/diag.sh b/tests/diag.sh index 1caf529f..2f307750 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -5,12 +5,12 @@ # not always able to convey back states to the upper-level test driver # begun 2009-05-27 by rgerhards # This file is part of the rsyslog project, released under GPLv3 -#valgrind="valgrind --log-fd=1" +#valgrind="valgrind --malloc-fill=ff --free-fill=fe --log-fd=1" #valgrind="valgrind --tool=drd --log-fd=1" #valgrind="valgrind --tool=helgrind --log-fd=1" #valgrind="valgrind --tool=exp-ptrcheck --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUG="debug nostdout" #export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason @@ -19,17 +19,18 @@ case $1 in rm -f rsyslog.action.*.include rm -f rsyslogd.started work-*.conf rsyslog.random.data rm -f rsyslogd2.started work-*.conf - rm -f work rsyslog.out.log rsyslog.out.log.save # common work files - rm -f rsyslog.out.*.log work-presort - rm -rf test-spool + rm -f work rsyslog.out.log rsyslog2.out.log rsyslog.out.log.save # common work files + rm -rf test-spool test-logdir + rm -f rsyslog.out.*.log work-presort rsyslog.pipe rm -f core.* vgcore.* mkdir test-spool ;; 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf rm -f rsyslogd2.started diag-common2.conf rsyslog.action.*.include - rm -f work rsyslog.out.log rsyslog.out.log.save # common work files - rm -f rsyslog.out.*.log rsyslog.random.data work-presort - rm -rf test-spool + rm -f work rsyslog.out.log rsyslog2.out.log rsyslog.out.log.save # common work files + rm -rf test-spool test-logdir + rm -f rsyslog.out.*.log rsyslog.random.data work-presort rsyslog.pipe + echo ------------------------------------------------------------------------------- ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use # returns only after successful startup, $3 is the instance (blank or 2!) @@ -50,6 +51,12 @@ case $1 in while test -f rsyslog$2.pid; do ./msleep 100 # wait 100 milliseconds done + if [ -e core.* ] + then + echo "ABORT! core file exists, starting interactive shell" + bash + exit 1 + fi ;; 'wait-queueempty') # wait for main message queue to be empty. $2 is the instance. if [ "$2" == "2" ] @@ -95,7 +102,8 @@ case $1 in cp rsyslog.out.log work-presort sort < rsyslog.out.log > work # $4... are just to have the abilit to pass in more options... - ./chkseq -fwork -v -s$2 -e$3 $4 $5 $6 $7 + # add -v to chkseq if you need more verbose output + ./chkseq -fwork -s$2 -e$3 $4 $5 $6 $7 if [ "$?" -ne "0" ]; then echo "sequence error detected" exit 1 @@ -107,11 +115,13 @@ case $1 in rm -f work2 sort < rsyslog2.out.log > work2 # $4... are just to have the abilit to pass in more options... - ./chkseq -fwork2 -v -s$2 -e$3 $4 $5 $6 $7 + # add -v to chkseq if you need more verbose output + ./chkseq -fwork2 -s$2 -e$3 $4 $5 $6 $7 if [ "$?" -ne "0" ]; then echo "sequence error detected" exit 1 fi + rm -f work2 ;; 'gzip-seq-check') # do the usual sequence check, but for gzip files rm -f work @@ -127,7 +137,7 @@ case $1 in ;; 'nettester') # perform nettester-based tests # use -v for verbose output! - ./nettester -t$2 -i$3 + ./nettester -t$2 -i$3 $4 if [ "$?" -ne "0" ]; then exit 1 fi diff --git a/tests/dircreate_dflt.sh b/tests/dircreate_dflt.sh new file mode 100755 index 00000000..71a671f3 --- /dev/null +++ b/tests/dircreate_dflt.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_dflt_dflt.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_dflt.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ ! -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile not created!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/dircreate_off.sh b/tests/dircreate_off.sh new file mode 100755 index 00000000..92fdee01 --- /dev/null +++ b/tests/dircreate_off.sh @@ -0,0 +1,20 @@ +# Test for automatic creation of dynafile directories +# note that we use the "test-spool" directory, because it is handled by diag.sh +# in any case, so we do not need to add any extra new test dir. +# added 2009-11-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =================================================================================== +echo \[dircreate_off_off.sh\]: testing automatic directory creation for dynafiles - default +source $srcdir/diag.sh init +source $srcdir/diag.sh startup dircreate_off.conf +source $srcdir/diag.sh injectmsg 0 1 # a single message is sufficient +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +if [ -e test-logdir/rsyslog.out.log ] +then + echo "test-logdir or logfile WAS created where not permitted to!" + exit 1 +fi +exit +source $srcdir/diag.sh exit diff --git a/tests/discard.sh b/tests/discard.sh new file mode 100755 index 00000000..96006f01 --- /dev/null +++ b/tests/discard.sh @@ -0,0 +1,17 @@ +# Test for discard functionality +# This test checks if discard works. It is not a perfect test but +# will find at least segfaults and obviously not discarded messages. +# added 2009-07-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo =============================================================================== +echo \[discard.sh\]: testing discard functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup discard.conf +# 20000 messages should be enough - the disk test is slow enough ;) +sleep 4 +source $srcdir/diag.sh tcpflood -m10 -i1 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 2 10 +source $srcdir/diag.sh exit diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh index c7745930..fe923c22 100755 --- a/tests/diskqueue-fsync.sh +++ b/tests/diskqueue-fsync.sh @@ -5,11 +5,11 @@ # added 2009-06-09 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -echo testing queue disk-only mode, fsync case +echo \[diskqueue-fsync.sh\]: testing queue disk-only mode, fsync case source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue-fsync.conf # 1000 messages should be enough - the disk fsync test is very slow! -source $srcdir/diag.sh tcpflood -m1000 +source $srcdir/diag.sh injectmsg 0 1000 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages source $srcdir/diag.sh wait-shutdown source $srcdir/diag.sh seq-check 0 999 diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 7a50d82e..b871e9eb 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -5,7 +5,7 @@ # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -echo diskqueue.sh: testing queue disk-only mode +echo \[diskqueue.sh\]: testing queue disk-only mode # uncomment for debugging support: #export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" #export RSYSLOG_DEBUGLOG="log" diff --git a/tests/execonlyonce.sh b/tests/execonlyonce.sh index b7f60849..8e184070 100755 --- a/tests/execonlyonce.sh +++ b/tests/execonlyonce.sh @@ -16,6 +16,7 @@ sleep 4 # one more than the once inerval! # and inject another couple of messages source $srcdir/diag.sh tcpflood -m10 -i100 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # now we need your custom logic to see if the result is equal to the # expected result diff --git a/tests/execonlywhenprevsuspended.sh b/tests/execonlywhenprevsuspended.sh new file mode 100755 index 00000000..624f64af --- /dev/null +++ b/tests/execonlywhenprevsuspended.sh @@ -0,0 +1,13 @@ +# we test the execonly if previous is suspended directive. This is the +# most basic test which soley tests a singel case but no dependencies within +# the ruleset. +# rgerhards, 2010-06-23 +echo ===================================================================================== +echo \[execonlywhenprevsuspended.sh\]: test execonly...suspended functionality simple case +source $srcdir/diag.sh init +source $srcdir/diag.sh startup execonlywhenprevsuspended.conf +source $srcdir/diag.sh injectmsg 0 1000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 1 999 +source $srcdir/diag.sh exit diff --git a/tests/execonlywhenprevsuspended2.sh b/tests/execonlywhenprevsuspended2.sh new file mode 100755 index 00000000..8af1b4d8 --- /dev/null +++ b/tests/execonlywhenprevsuspended2.sh @@ -0,0 +1,17 @@ +# we test the execonly if previous is suspended directive. For this, +# we have an action that is suspended for all messages but the second. +# we write two files: one only if the output is suspended and the other one +# in all cases. This should thouroughly check the logic involved. +# rgerhards, 2010-06-23 +echo =============================================================================== +echo \[execonlywhenprevsuspended2.sh\]: test execonly...suspended functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup execonlywhenprevsuspended2.conf +source $srcdir/diag.sh injectmsg 0 1000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +echo check file 1 +source $srcdir/diag.sh seq-check 1 999 +echo check file 2 +source $srcdir/diag.sh seq-check2 0 999 +source $srcdir/diag.sh exit diff --git a/tests/execonlywhenprevsuspended3.sh b/tests/execonlywhenprevsuspended3.sh new file mode 100755 index 00000000..408aeba4 --- /dev/null +++ b/tests/execonlywhenprevsuspended3.sh @@ -0,0 +1,17 @@ +# we test the execonly if previous is suspended directive. +# This test checks if, within the same rule, one action can be set +# to emit only if the previous was suspended while the next action +# always sends data. +# rgerhards, 2010-06-24 +echo =============================================================================== +echo \[execonlywhenprevsuspended3.sh\]: test execonly...suspended functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup execonlywhenprevsuspended3.conf +source $srcdir/diag.sh injectmsg 0 1000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +echo check file 1 +source $srcdir/diag.sh seq-check 1 999 +echo check file 2 +source $srcdir/diag.sh seq-check2 0 999 +source $srcdir/diag.sh exit diff --git a/tests/execonlywhenprevsuspended4.sh b/tests/execonlywhenprevsuspended4.sh new file mode 100755 index 00000000..87008b33 --- /dev/null +++ b/tests/execonlywhenprevsuspended4.sh @@ -0,0 +1,16 @@ +# we test the execonly if previous is suspended directive. +# This test checks if multiple backup actions can be defined. +# rgerhards, 2010-06-24 +echo =============================================================================== +echo \[execonlywhenprevsuspended4.sh\]: test execonly..suspended multi backup action +source $srcdir/diag.sh init +source $srcdir/diag.sh startup execonlywhenprevsuspended4.conf +source $srcdir/diag.sh injectmsg 0 1000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 1 999 +if [[ -s rsyslog2.out.log ]] ; then + echo failure: second output file has data where it should be empty + exit 1 +fi ; +source $srcdir/diag.sh exit diff --git a/tests/fieldtest.sh b/tests/fieldtest.sh index 482fa143..9875fda6 100755 --- a/tests/fieldtest.sh +++ b/tests/fieldtest.sh @@ -1,4 +1,4 @@ -echo test fieldtest via udp +echo \[fieldtest.sh\]: test fieldtest via udp $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tfield1 -iudp diff --git a/tests/inputname.sh b/tests/inputname.sh index e1a58517..71f11c1e 100755 --- a/tests/inputname.sh +++ b/tests/inputname.sh @@ -1,4 +1,4 @@ -echo testing $InputTCPServerInputName directive +echo \[inputname.sh\]: testing $InputTCPServerInputName directive $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason echo port 12514 diff --git a/tests/linkedlistqueue.sh b/tests/linkedlistqueue.sh new file mode 100755 index 00000000..e6d48a68 --- /dev/null +++ b/tests/linkedlistqueue.sh @@ -0,0 +1,17 @@ +# Test for Linkedlist queue mode +# added 2009-05-20 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo \[linkedlistqueue.sh\]: testing queue Linkedlist queue mode +source $srcdir/diag.sh init +source $srcdir/diag.sh startup linkedlistqueue.conf + +# 40000 messages should be enough +source $srcdir/diag.sh injectmsg 0 40000 + +# terminate *now* (don't wait for queue to drain) +kill `cat rsyslog.pid` + +# now wait until rsyslog.pid is gone (and the process finished) +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 94a5c035..ec8f2453 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,4 +1,5 @@ # test many concurrent tcp connections +echo \[manytcp.sh\]: test concurrent tcp connections source $srcdir/diag.sh init source $srcdir/diag.sh startup manytcp.conf # the config file specifies exactly 1100 connections diff --git a/tests/nettester.c b/tests/nettester.c index 24e20422..7c8c413a 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -47,6 +47,7 @@ #include <signal.h> #include <netinet/in.h> #include <getopt.h> +#include <errno.h> #include <ctype.h> #define EXIT_FAILURE 1 @@ -62,6 +63,7 @@ static char *testSuite = NULL; /* name of current test suite */ static int iPort = 12514; /* port which shall be used for sending data */ static char* pszCustomConf = NULL; /* custom config file, use -c conf to specify */ static int verbose = 0; /* verbose output? -v option */ +static int IPv4Only = 0; /* use only IPv4 in rsyslogd call? */ static char **ourEnvp; /* these two are quick hacks... */ @@ -92,6 +94,7 @@ void readLine(int fd, char *ln) if(verbose) fprintf(stderr, "begin readLine\n"); lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { if(c == '\0') { *ln = c; @@ -104,6 +107,11 @@ void readLine(int fd, char *ln) } *ln = '\0'; + if(lenRead < 0) { + printf("read from rsyslogd returned with error '%s' - aborting test\n", strerror(errno)); + exit(1); + } + if(verbose) fprintf(stderr, "end readLine, val read '%s'\n", orig); } @@ -157,7 +165,7 @@ tcpSend(char *buf, int lenBuf) iRet = 1; goto finalize_it; } else { - usleep(100000); /* 0.1 sec, these are us! */ + usleep(100000); /* ms = 1000 us! */ } } } @@ -236,13 +244,16 @@ int openPipe(char *configFile, pid_t *pid, int *pfd) int pipefd[2]; pid_t cpid; char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime/.libs:../.libs", NULL }; + "-M../runtime/.libs:../.libs", NULL, NULL}; char confFile[1024]; sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, (pszCustomConf == NULL) ? configFile : pszCustomConf); newargv[1] = confFile; + if(IPv4Only) + newargv[(sizeof(newargv)/sizeof(char*)) - 2] = "-4"; + if (pipe(pipefd) == -1) { perror("pipe"); exit(EXIT_FAILURE); @@ -384,12 +395,17 @@ processTestFile(int fd, char *pszFileName) /* pull response from server and then check if it meets our expectation */ //printf("try pull pipe...\n"); readLine(fd, buf); + if(strlen(buf) == 0) { + printf("something went wrong - read a zero-length string from rsyslogd\n"); + exit(1); + } if(strcmp(expected, buf)) { ++iFailed; printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", expected, buf); ret = 1; } + /* we need to free buffers, as we have potentially modified them! */ free(testdata); testdata = NULL; @@ -397,7 +413,6 @@ processTestFile(int fd, char *pszFileName) expected = NULL; } - free(expected); fclose(fp); return(ret); } @@ -452,11 +467,24 @@ doTests(int fd, char *files) return(iFailed); } + +/* indicate that our child has died (where it is not permitted to!). + */ +void childDied(__attribute__((unused)) int sig) +{ + printf("ERROR: child died unexpectedly (maybe a segfault?)!\n"); + exit(1); +} + + /* cleanup */ void doAtExit(void) { int status; + /* disarm died-child handler */ + signal(SIGCHLD, SIG_IGN); + if(rsyslogdPid != 0) { kill(rsyslogdPid, SIGTERM); waitpid(rsyslogdPid, &status, 0); /* wait until instance terminates */ @@ -480,8 +508,11 @@ int main(int argc, char *argv[], char *envp[]) char testcases[4096]; ourEnvp = envp; - while((opt = getopt(argc, argv, "dc:i:p:t:v")) != EOF) { + while((opt = getopt(argc, argv, "4c:i:p:t:v")) != EOF) { switch((char)opt) { + case '4': + IPv4Only = 1; + break; case 'c': pszCustomConf = optarg; break; @@ -538,6 +569,9 @@ int main(int argc, char *argv[], char *envp[]) } fclose(fp); + /* arm died-child handler */ + signal(SIGCHLD, childDied); + /* make sure we do not abort if there is an issue with pipes. * our code does the necessary error handling. */ @@ -553,5 +587,6 @@ int main(int argc, char *argv[], char *envp[]) ret = 1; if(verbose) printf("End of nettester run (%d).\n", ret); + exit(ret); } diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index 2c2a8ef3..4e916f1e 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,4 +1,4 @@ -echo test omod-if-array via udp +echo \[omod-if-array.sh\]: test omod-if-array via udp $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tomod-if-array -iudp -p4711 diff --git a/tests/omruleset-queue.sh b/tests/omruleset-queue.sh new file mode 100755 index 00000000..cfb80c62 --- /dev/null +++ b/tests/omruleset-queue.sh @@ -0,0 +1,19 @@ +# test for omruleset. What we do is have the main queue forward +# all messages to a secondary ruleset via omruleset, which then does +# the actual file write. We check if all messages arrive at the file, +# what implies that omruleset works. No filters or special queue modes +# are used, but the ruleset uses its own queue. So we can also inject +# more messages without running into troubles. +# added 2009-11-02 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[omruleset-queue.sh\]: test for omruleset functionality with a ruleset queue +source $srcdir/diag.sh init +source $srcdir/diag.sh startup omruleset-queue.conf +source $srcdir/diag.sh injectmsg 0 20000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 19999 +source $srcdir/diag.sh exit diff --git a/tests/omruleset.sh b/tests/omruleset.sh new file mode 100755 index 00000000..dbc5cb31 --- /dev/null +++ b/tests/omruleset.sh @@ -0,0 +1,22 @@ +# Basic test for omruleset. What we do is have the main queue forward +# all messages to a secondary ruleset via omruleset, which then does +# the actual file write. We check if all messages arrive at the file, +# what implies that omruleset works. No filters or special queue modes +# are used, so the message is re-enqueued into the main message queue. +# We inject just 5,000 message because we may otherwise run into +# queue full conditions (as we use the same queue) and that +# would result in longer execution time. In any case, 5000 messages +# are well enough to test what we want to test. +# added 2009-11-02 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[omruleset.sh\]: basic test for omruleset functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup omruleset.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/parsertest.sh b/tests/parsertest.sh index fc68ab84..9f9c2f74 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -2,6 +2,10 @@ echo TEST: \[parsertest.sh\]: various parser tests source $srcdir/diag.sh init source $srcdir/diag.sh nettester parse1 udp source $srcdir/diag.sh nettester parse1 tcp +source $srcdir/diag.sh nettester parse2 udp +source $srcdir/diag.sh nettester parse2 tcp +source $srcdir/diag.sh nettester parse_8bit_escape udp +source $srcdir/diag.sh nettester parse_8bit_escape tcp source $srcdir/diag.sh nettester parse3 udp source $srcdir/diag.sh nettester parse3 tcp source $srcdir/diag.sh nettester parse_invld_regex udp @@ -18,6 +22,10 @@ source $srcdir/diag.sh nettester snare_ccoff_udp2 udp echo \[parsertest.sh]: redoing tests in IPv4-only mode source $srcdir/diag.sh nettester parse1 udp -4 source $srcdir/diag.sh nettester parse1 tcp -4 +source $srcdir/diag.sh nettester parse2 udp -4 +source $srcdir/diag.sh nettester parse2 tcp -4 +source $srcdir/diag.sh nettester parse_8bit_escape udp -4 +source $srcdir/diag.sh nettester parse_8bit_escape tcp -4 source $srcdir/diag.sh nettester parse3 udp -4 source $srcdir/diag.sh nettester parse3 tcp -4 source $srcdir/diag.sh nettester parse_invld_regex udp -4 diff --git a/tests/pipe_noreader.sh b/tests/pipe_noreader.sh new file mode 100755 index 00000000..b2c46581 --- /dev/null +++ b/tests/pipe_noreader.sh @@ -0,0 +1,29 @@ +# This is test driver for a pipe that has no reader. This mimics a usual +# real-world scenario, the /dev/xconsole pipe. Some versions of rsyslog +# were known to hang or loop on this pipe, thus we added this scenario +# as a permanent testcase. For some details, please see bug tracker +# http://bugzilla.adiscon.com/show_bug.cgi?id=186 +# +# IMPORTANT: we do NOT check any result message set. The whole point in +# this test is to verify that we do NOT run into an eternal loop. As such, +# the test is "PASS", if rsyslogd terminates. If it does not terminate, we +# obviously do not cause "FAIL", but processing will hang, which should be +# a good-enough indication of failure. +# +# added 2010-04-26 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo TEST: \[pipe_noreader.sh\]: test for pipe writing without reader +# uncomment for debugging support: +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="log" +source $srcdir/diag.sh init +mkfifo ./rsyslog.pipe +source $srcdir/diag.sh startup pipe_noreader.conf +# we need to emit ~ 128K of data according to bug report +source $srcdir/diag.sh tcpflood -m1000 -d500 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown # and wait for it to terminate +# NO need to check seqno -- see header comment +echo we did not loop, so the test is sucessfull +source $srcdir/diag.sh exit diff --git a/tests/pmlastmsg.sh b/tests/pmlastmsg.sh new file mode 100755 index 00000000..9a14ce21 --- /dev/null +++ b/tests/pmlastmsg.sh @@ -0,0 +1,6 @@ +echo ============================================================================== +echo \[pmlastmsg.sh\]: tests for pmlastmsg +source $srcdir/diag.sh init +source $srcdir/diag.sh nettester pmlastmsg udp +source $srcdir/diag.sh nettester pmlastmsg tcp +source $srcdir/diag.sh exit diff --git a/tests/proprepltest.sh b/tests/proprepltest.sh index 3c252e52..2e59a31d 100755 --- a/tests/proprepltest.sh +++ b/tests/proprepltest.sh @@ -1,4 +1,4 @@ -echo TEST: proprepltest.sh - various tests for the property replacer +echo \[proprepltest.sh\]: various tests for the property replacer source $srcdir/diag.sh init source $srcdir/diag.sh nettester rfctag udp source $srcdir/diag.sh nettester rfctag tcp diff --git a/tests/queue-persist-drvr.sh b/tests/queue-persist-drvr.sh index ea5386a7..53fbcb8b 100755 --- a/tests/queue-persist-drvr.sh +++ b/tests/queue-persist-drvr.sh @@ -24,5 +24,6 @@ source $srcdir/diag.sh check-mainq-spool echo "#" > work-delay.conf source $srcdir/diag.sh startup queue-persist.conf source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +$srcdir/diag.sh wait-shutdown source $srcdir/diag.sh seq-check 0 4999 source $srcdir/diag.sh exit diff --git a/tests/queue-persist.sh b/tests/queue-persist.sh index 999655b1..ff1842bb 100755 --- a/tests/queue-persist.sh +++ b/tests/queue-persist.sh @@ -2,6 +2,7 @@ # to carry out multiple tests with different queue modes # added 2009-05-27 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 +echo \[queue-persist.sh\]: source $srcdir/queue-persist-drvr.sh LinkedList source $srcdir/queue-persist-drvr.sh FixedArray # the disk test should not fail, however, the config is extreme and using diff --git a/tests/rulesetmultiqueue.sh b/tests/rulesetmultiqueue.sh new file mode 100755 index 00000000..71ed9dce --- /dev/null +++ b/tests/rulesetmultiqueue.sh @@ -0,0 +1,33 @@ +# Test for disk-only queue mode +# This tests defines three rulesets, each one with its own queue. Then, it +# sends data to them and checks the outcome. Note that we do need to +# use some custom code as the test driver framework does not (yet?) +# support multi-output-file operations. +# added 2009-10-30 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[rulesetmultiqueu.sh\]: testing multiple queues via rulesets +source $srcdir/diag.sh init +rm -f rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log +source $srcdir/diag.sh startup rulesetmultiqueue.conf +source $srcdir/diag.sh wait-startup +# now fill the three files (a bit sequentially, but they should +# still get their share of concurrency - to increase the chance +# we use three connections per set). +source $srcdir/diag.sh tcpflood -c3 -p13514 -m20000 -i0 +source $srcdir/diag.sh tcpflood -c3 -p13515 -m20000 -i20000 +source $srcdir/diag.sh tcpflood -c3 -p13516 -m20000 -i40000 + +# in this version of the imdiag, we do not have the capability to poll +# all queues for emptyness. So we do a sleep in the hopes that this will +# sufficiently drain the queues. This is race, but the best we currently +# can do... - rgerhards, 2009-11-05 +sleep 2 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +# now consolidate all logs into a single one so that we can use the +# regular check logic +cat rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 59999 +rm -f rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log +source $srcdir/diag.sh exit diff --git a/tests/runtime-dummy.c b/tests/runtime-dummy.c index 38e6bba1..5a9039bf 100644 --- a/tests/runtime-dummy.c +++ b/tests/runtime-dummy.c @@ -34,12 +34,13 @@ int repeatinterval = 30; int bActExecWhenPrevSusp = 0; int iActExecOnceInterval = 1; int MarkInterval = 30; +void *pMsgQueue = NULL; void cflineClassic(void) {}; void selectorAddList(void) {}; void selectorConstruct(void) {}; void selectorDestruct(void) {}; -void getFIOPName(void) {}; +rsRetVal createMainQueue(void) { return RS_RET_ERR; } ruleset_t *pCurrRuleset; /* these are required by some dynamically loaded modules */ diff --git a/tests/sndrcv_omudpspoof.sh b/tests/sndrcv_omudpspoof.sh new file mode 100755 index 00000000..bb804d93 --- /dev/null +++ b/tests/sndrcv_omudpspoof.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the standard +# ports. Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_omudpspoof.sh\]: testing sending and receiving via omudp +source $srcdir/sndrcv_drvr.sh sndrcv_omudpspoof 50 diff --git a/tests/sndrcv_omudpspoof_nonstdpt.sh b/tests/sndrcv_omudpspoof_nonstdpt.sh new file mode 100755 index 00000000..6aeb1a5f --- /dev/null +++ b/tests/sndrcv_omudpspoof_nonstdpt.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the standard +# ports. Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_omudpspoof_nonstdpt.sh\]: testing sending and receiving via omudp +source $srcdir/sndrcv_drvr.sh sndrcv_omudpspoof_nonstdpt 50 diff --git a/tests/sndrcv_udp.sh b/tests/sndrcv_udp.sh new file mode 100755 index 00000000..274a414a --- /dev/null +++ b/tests/sndrcv_udp.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the standard +# ports. Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_udp.sh\]: testing sending and receiving via udp +source $srcdir/sndrcv_drvr.sh sndrcv_udp 50 diff --git a/tests/sndrcv_udp_nonstdpt.sh b/tests/sndrcv_udp_nonstdpt.sh new file mode 100755 index 00000000..2ad2906f --- /dev/null +++ b/tests/sndrcv_udp_nonstdpt.sh @@ -0,0 +1,10 @@ +# This runs sends and receives messages via UDP to the non-standard port 2514 +# Note that with UDP we can always have message loss. While this is +# less likely in a local environment, we strongly limit the amount of data +# we send in the hope to not lose any messages. However, failure of this +# test does not necessarily mean that the code is wrong (but it is very likely!) +# added 2009-11-11 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[sndrcv_udp_nonstdpt.sh\]: testing sending and receiving via udp +source $srcdir/sndrcv_drvr.sh sndrcv_udp_nonstdpt 50 diff --git a/tests/tabescape_dflt.sh b/tests/tabescape_dflt.sh new file mode 100755 index 00000000..d0e13ec9 --- /dev/null +++ b/tests/tabescape_dflt.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_dflt.sh\]: test for default tab escaping +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_dflt -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_dflt -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/tabescape_off.sh b/tests/tabescape_off.sh new file mode 100755 index 00000000..71ede7c0 --- /dev/null +++ b/tests/tabescape_off.sh @@ -0,0 +1,14 @@ +echo =============================================================================== +echo \[tabescape_off.sh\]: test for tab escaping off +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -ttabescape_off -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test via tcp +./nettester -ttabescape_off -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 2d8be10f..fc93903a 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -5,6 +5,7 @@ * -t target address (default 127.0.0.1) * -p target port (default 13514) * -n number of target ports (targets are in range -p..(-p+-n-1) + * Note -c must also be set to at LEAST the number of -n! * -c number of connections (default 1) * -m number of messages to send (connection is random) * -i initial message number (optional) @@ -447,10 +448,12 @@ int main(int argc, char *argv[]) exit(1); } + closeConnections(); /* this is important so that we do not finish too early! */ + if(nConnDrops > 0) printf("-D option initiated %ld connection closures\n", nConnDrops); + printf("End of tcpflood Run\n"); - closeConnections(); /* this is important so that we do not finish too early! */ exit(ret); } diff --git a/tests/testsuites/1.tabescape_dflt b/tests/testsuites/1.tabescape_dflt new file mode 100644 index 00000000..91444bd3 --- /dev/null +++ b/tests/testsuites/1.tabescape_dflt @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT#011after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.tabescape_off b/tests/testsuites/1.tabescape_off new file mode 100644 index 00000000..6a331c35 --- /dev/null +++ b/tests/testsuites/1.tabescape_off @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 test: before HT after HT (do NOT remove TAB!) + before HT after HT (do NOT remove TAB!) +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/4.parse1 b/tests/testsuites/4.parse1 new file mode 100644 index 00000000..07e2445a --- /dev/null +++ b/tests/testsuites/4.parse1 @@ -0,0 +1,4 @@ +<29>Jul 31 21:39:21 example-b example-gw[10538]: disconnect host=/192.0.2.1 destination=192.0.2.2/11282 in=3274 out=1448 duration=0 +29,daemon,notice,Jul 31 21:39:21,example-b,example-gw,example-gw[10538]:, disconnect host=/192.0.2.1 destination=192.0.2.2/11282 in=3274 out=1448 duration=0 +# yet another real-life sample where we had some issues with - the important +# part is the dash inside the hostname! diff --git a/tests/testsuites/8bit.parse1 b/tests/testsuites/8bit.parse1 new file mode 100644 index 00000000..90db6352 --- /dev/null +++ b/tests/testsuites/8bit.parse1 @@ -0,0 +1,2 @@ +<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü +6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: äöü diff --git a/tests/testsuites/8bit.parse_8bit_escape b/tests/testsuites/8bit.parse_8bit_escape new file mode 100644 index 00000000..b2f6335c --- /dev/null +++ b/tests/testsuites/8bit.parse_8bit_escape @@ -0,0 +1,2 @@ +<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü +6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: #303#244#303#266#303#274 diff --git a/tests/testsuites/arrayqueue.conf b/tests/testsuites/arrayqueue.conf new file mode 100644 index 00000000..c5874a83 --- /dev/null +++ b/tests/testsuites/arrayqueue.conf @@ -0,0 +1,14 @@ +# Test for queue fixedArray mode (see .sh file for details) +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +# set spool locations and switch queue to disk-only mode +$MainMsgQueueType FixedArray + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/badqi.conf b/tests/testsuites/badqi.conf new file mode 100644 index 00000000..0ab059df --- /dev/null +++ b/tests/testsuites/badqi.conf @@ -0,0 +1,15 @@ +# Test for bad .qi file (see .sh file for details) +# rgerhards, 2009-10-21 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +# instruct to use bad .qi file +$WorkDirectory bad_qi +$ActionQueueType LinkedList +$ActionQueueFileName dbq +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf new file mode 100644 index 00000000..843a3e4f --- /dev/null +++ b/tests/testsuites/da-mainmsg-q.conf @@ -0,0 +1,21 @@ +# Test for DA mode in main message queue (see .sh file for details) +# rgerhards, 2009-04-22 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk assisted mode +$WorkDirectory test-spool +$MainMsgQueueSize 200 # this *should* trigger moving on to DA mode... +# note: we must set QueueSize sufficiently high, so that 70% (light delay mark) +# is high enough above HighWatermark! +$MainMsgQueueHighWatermark 80 +$MainMsgQueueLowWatermark 40 +$MainMsgQueueFilename mainq +$MainMsgQueueType linkedlist + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/dircreate_dflt.conf b/tests/testsuites/dircreate_dflt.conf new file mode 100644 index 00000000..9b9aadb8 --- /dev/null +++ b/tests/testsuites/dircreate_dflt.conf @@ -0,0 +1,11 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/dircreate_off.conf b/tests/testsuites/dircreate_off.conf new file mode 100644 index 00000000..28ccbd8c --- /dev/null +++ b/tests/testsuites/dircreate_off.conf @@ -0,0 +1,12 @@ +# see .sh file for description +# rgerhards, 2009-11-30 +$IncludeConfig diag-common.conf + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$CreateDirs off +$template dynfile,"test-logdir/rsyslog.out.log" # trick to use relative path names! +*.* ?dynfile diff --git a/tests/testsuites/discard.conf b/tests/testsuites/discard.conf new file mode 100644 index 00000000..bbe2fe77 --- /dev/null +++ b/tests/testsuites/discard.conf @@ -0,0 +1,13 @@ +# Test for discard functionality +# rgerhards, 2009-07-30 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +:msg, contains, "00000001" ~ + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/empty.parse1 b/tests/testsuites/empty.parse1 new file mode 100644 index 00000000..86a86986 --- /dev/null +++ b/tests/testsuites/empty.parse1 @@ -0,0 +1,3 @@ +<14>Jan 6 2009 15:22:26 localhost +14,user,info,Jan 6 15:22:26,localhost,,, +#Note: there is one space after localhost, but then \n! diff --git a/tests/testsuites/execonlywhenprevsuspended.conf b/tests/testsuites/execonlywhenprevsuspended.conf new file mode 100644 index 00000000..04dc6b59 --- /dev/null +++ b/tests/testsuites/execonlywhenprevsuspended.conf @@ -0,0 +1,13 @@ +# See main .sh file for info +# rgerhards, 2010-06-23 +$IncludeConfig diag-common.conf + +# omtesting provides the ability to cause "SUSPENDED" action state +$ModLoad ../plugins/omtesting/.libs/omtesting + +$MainMsgQueueTimeoutShutdown 100000 +$template outfmt,"%msg:F,58:2%\n" + +:msg, contains, "msgnum:" :omtesting:fail 2 0 +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/execonlywhenprevsuspended2.conf b/tests/testsuites/execonlywhenprevsuspended2.conf new file mode 100644 index 00000000..86aa8832 --- /dev/null +++ b/tests/testsuites/execonlywhenprevsuspended2.conf @@ -0,0 +1,19 @@ +# See main .sh file for info +# rgerhards, 2010-06-23 +$IncludeConfig diag-common.conf + +# omtesting provides the ability to cause "SUSPENDED" action state +$ModLoad ../plugins/omtesting/.libs/omtesting + +$MainMsgQueueTimeoutShutdown 100000 +$template outfmt,"%msg:F,58:2%\n" + +:msg, contains, "msgnum:" :omtesting:fail 2 0 +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log;outfmt +# note that we MUST re-set PrevSusp, else it will remain active +# for all other actions as well (this tells us how bad the current +# config language is...). -- rgerhards, 2010-06-24 +$ActionExecOnlyWhenPreviousIsSuspended off + +:msg, contains, "msgnum:" ./rsyslog2.out.log;outfmt diff --git a/tests/testsuites/execonlywhenprevsuspended3.conf b/tests/testsuites/execonlywhenprevsuspended3.conf new file mode 100644 index 00000000..d2750e9a --- /dev/null +++ b/tests/testsuites/execonlywhenprevsuspended3.conf @@ -0,0 +1,18 @@ +# See main .sh file for info +# rgerhards, 2010-06-23 +$IncludeConfig diag-common.conf + +# omtesting provides the ability to cause "SUSPENDED" action state +$ModLoad ../plugins/omtesting/.libs/omtesting + +$MainMsgQueueTimeoutShutdown 100000 +$template outfmt,"%msg:F,58:2%\n" + +:msg, contains, "msgnum:" :omtesting:fail 2 0 +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log;outfmt +# note that we MUST re-set PrevSusp, else it will remain active +# for all other actions as well (this tells us how bad the current +# config language is...). -- rgerhards, 2010-06-24 +$ActionExecOnlyWhenPreviousIsSuspended off +& ./rsyslog2.out.log;outfmt diff --git a/tests/testsuites/execonlywhenprevsuspended4.conf b/tests/testsuites/execonlywhenprevsuspended4.conf new file mode 100644 index 00000000..04bc3805 --- /dev/null +++ b/tests/testsuites/execonlywhenprevsuspended4.conf @@ -0,0 +1,15 @@ +# See main .sh file for info +# rgerhards, 2010-06-23 +$IncludeConfig diag-common.conf + +# omtesting provides the ability to cause "SUSPENDED" action state +$ModLoad ../plugins/omtesting/.libs/omtesting + +$MainMsgQueueTimeoutShutdown 100000 +$template outfmt,"%msg:F,58:2%\n" + +:msg, contains, "msgnum:" :omtesting:fail 2 0 +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log;outfmt +# note that $ActionExecOnlyWhenPreviousIsSuspended on is still active! +& ./rsyslog2.out.log;outfmt diff --git a/tests/testsuites/linkedlistqueue.conf b/tests/testsuites/linkedlistqueue.conf new file mode 100644 index 00000000..92a9649c --- /dev/null +++ b/tests/testsuites/linkedlistqueue.conf @@ -0,0 +1,16 @@ +# Test for queue LinkedList mode (see .sh file for details) +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$MainMsgQueueType LinkedList + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/malformed1.parse1 b/tests/testsuites/malformed1.parse1 new file mode 100644 index 00000000..a8825fe8 --- /dev/null +++ b/tests/testsuites/malformed1.parse1 @@ -0,0 +1,6 @@ +<131>Oct 8 23:05:06 10.321.1.123 05",result_code=200,b +131,local0,err,Oct 8 23:05:06,10.321.1.123,05",result_code=200,b,05",result_code=200,b, +# a somewhat mangeld-with real-life sample of a malformed message +# the key here is not what is being parsed, but that we do not abort! +# NOTE: if a parser enhancement breaks the format, this is probably OK +# also note that the above message does NOT contain a MSG part diff --git a/tests/testsuites/mark.parse1 b/tests/testsuites/mark.parse1 new file mode 100644 index 00000000..fff9ae6d --- /dev/null +++ b/tests/testsuites/mark.parse1 @@ -0,0 +1,7 @@ +#This is a malformed message, but one from real life. At least, +#it should be parsed as can be seen here. +<6>Feb 18 16:01:59 serverX -- MARK -- +6,kern,info,Feb 18 16:01:59,serverX,--,--, MARK -- +# and the next one as an extreme case (note the absence of PRI) +Feb 18 16:01:59 serverX -- MARK -- +13,user,notice,Feb 18 16:01:59,serverX,--,--, MARK -- diff --git a/tests/testsuites/master.pmlastmsg b/tests/testsuites/master.pmlastmsg new file mode 100644 index 00000000..170538b0 --- /dev/null +++ b/tests/testsuites/master.pmlastmsg @@ -0,0 +1,28 @@ +# the following messages should be processed by pmlastmsg: +<13>last message repeated 5 times +last message repeated 5 times +# +<13>last message repeated 0090909787348927349875 times +last message repeated 0090909787348927349875 times +# now slightly malformed formats that should NOT be processed +# by pmlasmsg: +<13>last message repeated 5 times + repeated 5 times +# +<13>last message repeated 5 times -- more data + repeated 5 times -- more data +# message count invalid: +<13>last message repeated 5.2 times + repeated 5.2 times +# +# +# now follow samples of non-pmlastmsg messages: +# +<167>Mar 6 16:57:54 172.20.245.8 TAG: Rest of message... + Rest of message... +# Now exactly with 32 characters +<167>Mar 6 16:57:54 172.20.245.8 TAG long message ================================================================================ + long message ================================================================================ +# RFC5424 messages +<34>1 2003-11-11T22:14:15.003Z mymachine.example.com su - ID47 last message repeated 5 times +last message repeated 5 times diff --git a/tests/testsuites/omruleset-queue.conf b/tests/testsuites/omruleset-queue.conf new file mode 100644 index 00000000..edc33c40 --- /dev/null +++ b/tests/testsuites/omruleset-queue.conf @@ -0,0 +1,20 @@ +# test for omruleset (see .sh file for details) +# rgerhards, 2009-11-02 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/omruleset/.libs/omruleset +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +$ruleset rsinclude +# create ruleset main queue with default parameters +$RulesetCreateMainQueue on +# make sure we do not terminate too early! +$MainMsgQueueTimeoutShutdown 10000 +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +$ruleset RSYSLOG_DefaultRuleset +$ActionOmrulesetRulesetName rsinclude +*.* :omruleset: diff --git a/tests/testsuites/omruleset.conf b/tests/testsuites/omruleset.conf new file mode 100644 index 00000000..fa6b906e --- /dev/null +++ b/tests/testsuites/omruleset.conf @@ -0,0 +1,16 @@ +# Basic test for omruleset (see .sh file for details) +# rgerhards, 2009-11-02 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/omruleset/.libs/omruleset +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +$ruleset rsinclude +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +$ruleset RSYSLOG_DefaultRuleset +$ActionOmrulesetRulesetName rsinclude +*.* :omruleset: diff --git a/tests/testsuites/parse2.conf b/tests/testsuites/parse2.conf new file mode 100644 index 00000000..04d910bc --- /dev/null +++ b/tests/testsuites/parse2.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template output,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;output diff --git a/tests/testsuites/parse_8bit_escape.conf b/tests/testsuites/parse_8bit_escape.conf new file mode 100644 index 00000000..0598f33f --- /dev/null +++ b/tests/testsuites/parse_8bit_escape.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off +$Escape8BitCharactersOnReceive on + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/pipe_noreader.conf b/tests/testsuites/pipe_noreader.conf new file mode 100644 index 00000000..63997760 --- /dev/null +++ b/tests/testsuites/pipe_noreader.conf @@ -0,0 +1,10 @@ +# simple async writing test +# rgerhards, 2010-03-09 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$template outfmt,"%msg:F,58:2%\n" +:msg, contains, "msgnum:" |./rsyslog.pipe diff --git a/tests/testsuites/pmlastmsg.conf b/tests/testsuites/pmlastmsg.conf new file mode 100644 index 00000000..59142dca --- /dev/null +++ b/tests/testsuites/pmlastmsg.conf @@ -0,0 +1,16 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/pmlastmsg/.libs/pmlastmsg +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# in this test, we use pmlastmsg, followed by the one-size-fits-all +# pm3164 (built-in) module. So we can test if non-pmlastmsg messages +# are also correctly detected. +$RulesetParser rsyslog.lastline +$RulesetParser rsyslog.rfc5424 +$RulesetParser rsyslog.rfc3164 + +# use a special format +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/reallife.parse1 b/tests/testsuites/reallife.parse1 new file mode 100644 index 00000000..a83d2dca --- /dev/null +++ b/tests/testsuites/reallife.parse1 @@ -0,0 +1,12 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# This file contains a lot of real-life samples (of course mangled so +# that they can not be traced back to the original submitter). Note +# that IP addr 192.0.2.1 is specifically set aside for testing and +# documentation by IANA. +# rgerhards, 2009-10-19 +<29>Oct 16 20:47:24 example-p exam-pl[12345]: connect host= /192.0.2.1 +29,daemon,notice,Oct 16 20:47:24,example-p,exam-pl,exam-pl[12345]:, connect host= /192.0.2.1 diff --git a/tests/testsuites/reallife.parse2 b/tests/testsuites/reallife.parse2 new file mode 100644 index 00000000..c42f2526 --- /dev/null +++ b/tests/testsuites/reallife.parse2 @@ -0,0 +1,12 @@ +# New tests should be added to this file if there is no specific +# reason for not doing that. Initially, we could only handle one test +# case per file, but this restriction has been removed some time ago. +# So it is less troublesome (and easier to overlook) to have all related +# tests in a single file. +# This file contains a lot of real-life samples (of course mangled so +# that they can not be traced back to the original submitter). Note +# that IP addr 192.0.2.1 is specifically set aside for testing and +# documentation by IANA. +# rgerhards, 2009-10-19 +<175>Oct 16 23:47:31 #001 MSWinEventLog 0#011Security#01119023582#011Fri Oct 16 16:30:44 2009#011592#011Security#011rgabcde#011User#011Success Audit#011XSXSXSN01#011Detailed Tracking#011#0112572#01119013885 +175,local5,debug,Oct 16 23:47:31,#001,#001, MSWinEventLog 0#011Security#01119023582#011Fri Oct 16 16:30:44 2009#011592#011Security#011rgabcde#011User#011Success Audit#011XSXSXSN01#011Detailed Tracking#011#0112572#01119013885 diff --git a/tests/testsuites/rulesetmultiqueue.conf b/tests/testsuites/rulesetmultiqueue.conf new file mode 100644 index 00000000..c8a82dd6 --- /dev/null +++ b/tests/testsuites/rulesetmultiqueue.conf @@ -0,0 +1,34 @@ +# Test for multiple ruleset queues (see .sh file for details) +# rgerhards, 2009-10-30 +$IncludeConfig diag-common.conf +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 + +# general definition +$template outfmt,"%msg:F,58:2%\n" + +# create the individual rulesets +$ruleset file1 +$RulesetCreateMainQueue on +$template dynfile1,"rsyslog.out1.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile1;outfmt + +$ruleset file2 +$RulesetCreateMainQueue on +$template dynfile2,"rsyslog.out2.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile2;outfmt + +$ruleset file3 +$RulesetCreateMainQueue on +$template dynfile3,"rsyslog.out3.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile3;outfmt + +# start listeners and bind them to rulesets +$InputTCPServerBindRuleset file1 +$InputTCPServerRun 13514 + +$InputTCPServerBindRuleset file2 +$InputTCPServerRun 13515 + +$InputTCPServerBindRuleset file3 +$InputTCPServerRun 13516 diff --git a/tests/testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf new file mode 100644 index 00000000..65659f00 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 2514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf new file mode 100644 index 00000000..29a30145 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_nonstdpt_sender.conf @@ -0,0 +1,18 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +$ModLoad ../plugins/omudpspoof/.libs/omudpspoof +$template spoofaddr,"127.0.0.1" + +#begin action definition +$ActionOMUDPSpoofSourceNameTemplate spoofaddr +$ActionOMUDPSpoofTargetHost 127.0.0.1 +$ActionOMUDPSpoofTargetPort 2514 +$ActionOMUDPSpoofSourcePortStart 514 +$ActionOMUDPSpoofSourcePortEnd 514 +*.* :omudpspoof: diff --git a/tests/testsuites/sndrcv_omudpspoof_rcvr.conf b/tests/testsuites/sndrcv_omudpspoof_rcvr.conf new file mode 100644 index 00000000..e5401811 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_omudpspoof_sender.conf b/tests/testsuites/sndrcv_omudpspoof_sender.conf new file mode 100644 index 00000000..c0d25935 --- /dev/null +++ b/tests/testsuites/sndrcv_omudpspoof_sender.conf @@ -0,0 +1,17 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +$ModLoad ../plugins/omudpspoof/.libs/omudpspoof +$template spoofaddr,"127.0.0.1" + +#begin action definition +$ActionOMUDPSpoofSourceNameTemplate spoofaddr +$ActionOMUDPSpoofTargetHost 127.0.0.1 +$ActionOMUDPSpoofSourcePortStart 514 +$ActionOMUDPSpoofSourcePortEnd 514 +*.* :omudpspoof: diff --git a/tests/testsuites/sndrcv_sender.conf b/tests/testsuites/sndrcv_sender.conf index c874c068..f3d6ba53 100644 --- a/tests/testsuites/sndrcv_sender.conf +++ b/tests/testsuites/sndrcv_sender.conf @@ -3,6 +3,7 @@ $IncludeConfig diag-common2.conf $ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! $InputTCPServerRun 13514 *.* @@127.0.0.1:13515 diff --git a/tests/testsuites/sndrcv_udp_nonstdpt_rcvr.conf b/tests/testsuites/sndrcv_udp_nonstdpt_rcvr.conf new file mode 100644 index 00000000..65659f00 --- /dev/null +++ b/tests/testsuites/sndrcv_udp_nonstdpt_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 2514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_udp_nonstdpt_sender.conf b/tests/testsuites/sndrcv_udp_nonstdpt_sender.conf new file mode 100644 index 00000000..2975f938 --- /dev/null +++ b/tests/testsuites/sndrcv_udp_nonstdpt_sender.conf @@ -0,0 +1,9 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +*.* @127.0.0.1:2514 diff --git a/tests/testsuites/sndrcv_udp_rcvr.conf b/tests/testsuites/sndrcv_udp_rcvr.conf new file mode 100644 index 00000000..e5401811 --- /dev/null +++ b/tests/testsuites/sndrcv_udp_rcvr.conf @@ -0,0 +1,11 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-12 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imudp/.libs/imudp +# then SENDER sends to this port (not tcpflood!) +$UDPServerRun 514 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/sndrcv_udp_sender.conf b/tests/testsuites/sndrcv_udp_sender.conf new file mode 100644 index 00000000..28e913ef --- /dev/null +++ b/tests/testsuites/sndrcv_udp_sender.conf @@ -0,0 +1,9 @@ +# see equally-named shell file for details +# rgerhards, 2009-11-11 +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +*.* @127.0.0.1 diff --git a/tests/testsuites/tabescape_dflt.conf b/tests/testsuites/tabescape_dflt.conf new file mode 100644 index 00000000..b9d92a37 --- /dev/null +++ b/tests/testsuites/tabescape_dflt.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tabescape_off.conf b/tests/testsuites/tabescape_off.conf new file mode 100644 index 00000000..c1eca305 --- /dev/null +++ b/tests/testsuites/tabescape_off.conf @@ -0,0 +1,10 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +$EscapeControlCharacterTab off + +# use a special format that we can easily parse in expect +$template fmt,"%msg%\n" +*.* :omstdout:;fmt diff --git a/tests/threadingmq.sh b/tests/threadingmq.sh index bdb5f35e..ea5d7837 100755 --- a/tests/threadingmq.sh +++ b/tests/threadingmq.sh @@ -6,7 +6,7 @@ # in practice many threading bugs result in an abort rather quickly and these # should be covered by this test here. # rgerhards, 2009-06-26 -echo TEST: threadingmq.sh - main queue concurrency +echo \[threadingmq.sh\]: main queue concurrency source $srcdir/diag.sh init source $srcdir/diag.sh startup threadingmq.conf source $srcdir/diag.sh injectmsg 0 100000 diff --git a/tests/threadingmqaq.sh b/tests/threadingmqaq.sh index 0104be00..ae5d3568 100755 --- a/tests/threadingmqaq.sh +++ b/tests/threadingmqaq.sh @@ -6,7 +6,7 @@ # in practice many threading bugs result in an abort rather quickly and these # should be covered by this test here. # rgerhards, 2009-06-26 -echo TEST: threadingmqaq.sh - main/action queue concurrency +echo \[threadingmqaq.sh\]: main/action queue concurrency source $srcdir/diag.sh init source $srcdir/diag.sh startup threadingmqaq.conf #source $srcdir/diag.sh tcpflood -c2 -m100000 diff --git a/tests/timestamp.sh b/tests/timestamp.sh index 7699a4af..71416c33 100755 --- a/tests/timestamp.sh +++ b/tests/timestamp.sh @@ -1,4 +1,4 @@ -echo various timestamp tests +echo \[timestamp.sh\]: various timestamp tests source $srcdir/diag.sh init source $srcdir/diag.sh nettester ts3164 udp source $srcdir/diag.sh nettester ts3164 tcp diff --git a/tests/validation-run.sh b/tests/validation-run.sh index 2e922283..cc29482a 100755 --- a/tests/validation-run.sh +++ b/tests/validation-run.sh @@ -21,6 +21,7 @@ # # A copy of the GPL can be found in the file "COPYING" in this distribution. #set -x +echo \[validation-run.sh\]: testing configuraton validation echo "testing a failed configuration verification run" ../tools/rsyslogd -dn -u2 -c4 -N1 -f$srcdir/testsuites/invalid.conf -M../runtime/.libs:../.libs if [ $? -ne 1 ]; then @@ -5,7 +5,7 @@ * * File begun on 2007-12-14 by RGerhards * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -29,6 +29,7 @@ #include <stdlib.h> #include <string.h> #include <signal.h> +#include <errno.h> #include <pthread.h> #include <assert.h> @@ -36,6 +37,7 @@ #include "dirty.h" #include "linkedlist.h" #include "threads.h" +#include "srUtils.h" /* linked list of currently-known threads */ static linkedList_t llThrds; @@ -44,23 +46,20 @@ static linkedList_t llThrds; /* Construct a new thread object */ -static rsRetVal thrdConstruct(thrdInfo_t **ppThis) +static rsRetVal +thrdConstruct(thrdInfo_t **ppThis) { DEFiRet; thrdInfo_t *pThis; assert(ppThis != NULL); - if((pThis = calloc(1, sizeof(thrdInfo_t))) == NULL) - return RS_RET_OUT_OF_MEMORY; - - /* OK, we got the element, now initialize members that should - * not be zero-filled. - */ - pThis->mutTermOK = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); - pthread_mutex_init (pThis->mutTermOK, NULL); - + CHKmalloc(pThis = calloc(1, sizeof(thrdInfo_t))); + pthread_mutex_init(&pThis->mutThrd, NULL); + pthread_cond_init(&pThis->condThrdTerm, NULL); *ppThis = pThis; + +finalize_it: RETiRet; } @@ -77,13 +76,54 @@ static rsRetVal thrdDestruct(thrdInfo_t *pThis) if(pThis->bIsActive == 1) { thrdTerminate(pThis); } - free(pThis->mutTermOK); + pthread_mutex_destroy(&pThis->mutThrd); + pthread_cond_destroy(&pThis->condThrdTerm); free(pThis); RETiRet; } +/* terminate a thread via the non-cancel interface + * This is a separate function as it involves a bit more of code. + * rgerhads, 2009-10-15 + */ +static inline rsRetVal +thrdTerminateNonCancel(thrdInfo_t *pThis) +{ + struct timespec tTimeout; + int ret; + DEFiRet; + assert(pThis != NULL); + + DBGPRINTF("request term via SIGTTIN for input thread 0x%x\n", (unsigned) pThis->thrdID); + pThis->bShallStop = TRUE; + do { + d_pthread_mutex_lock(&pThis->mutThrd); + pthread_kill(pThis->thrdID, SIGTTIN); + timeoutComp(&tTimeout, 10); /* a fixed 10ms timeout, do after lock (may take long!) */ + ret = d_pthread_cond_timedwait(&pThis->condThrdTerm, &pThis->mutThrd, &tTimeout); + d_pthread_mutex_unlock(&pThis->mutThrd); + if(Debug) { + if(ret == ETIMEDOUT) { + dbgprintf("input thread term: had a timeout waiting on thread termination\n"); + } else if(ret == 0) { + dbgprintf("input thread term: thread returned normally and is terminated\n"); + } else { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + dbgprintf("input thread term: cond_wait returned with error %d: %s\n", + err, errStr); + } + } + } while(pThis->bIsActive); + DBGPRINTF("non-cancel input thread termination succeeded for thread 0x%x\n", (unsigned) pThis->thrdID); + + RETiRet; +} + + /* terminate a thread gracefully. */ rsRetVal thrdTerminate(thrdInfo_t *pThis) @@ -91,9 +131,14 @@ rsRetVal thrdTerminate(thrdInfo_t *pThis) DEFiRet; assert(pThis != NULL); - pthread_cancel(pThis->thrdID); - pthread_join(pThis->thrdID, NULL); /* wait for cancel to complete */ - pThis->bIsActive = 0; + if(pThis->bNeedsCancel) { + DBGPRINTF("request term via canceling for input thread 0x%x\n", (unsigned) pThis->thrdID); + pthread_cancel(pThis->thrdID); + pThis->bIsActive = 0; + } else { + thrdTerminateNonCancel(pThis); + } + pthread_join(pThis->thrdID, NULL); /* wait for input thread to complete */ /* call cleanup function, if any */ if(pThis->pAfterRun != NULL) @@ -132,6 +177,11 @@ static void* thrdStarter(void *arg) sigfillset(&sigSet); pthread_sigmask(SIG_BLOCK, &sigSet, NULL); + /* but ignore SIGTTN, which we (ab)use to signal the thread to shutdown -- rgerhards, 2009-07-20 */ + sigemptyset(&sigSet); + sigaddset(&sigSet, SIGTTIN); + pthread_sigmask(SIG_UNBLOCK, &sigSet, NULL); + /* setup complete, we are now ready to execute the user code. We will not * regain control until the user code is finished, in which case we terminate * the thread. @@ -139,6 +189,16 @@ static void* thrdStarter(void *arg) iRet = pThis->pUsrThrdMain(pThis); dbgprintf("thrdStarter: usrThrdMain 0x%lx returned with iRet %d, exiting now.\n", (unsigned long) pThis->thrdID, iRet); + + /* signal master control that we exit (we do the mutex lock mostly to + * keep the thread debugger happer, it would not really be necessary with + * the logic we employ...) + */ + pThis->bIsActive = 0; + d_pthread_mutex_lock(&pThis->mutThrd); + pthread_cond_signal(&pThis->condThrdTerm); + d_pthread_mutex_unlock(&pThis->mutThrd); + ENDfunc pthread_exit(0); } @@ -147,7 +207,7 @@ static void* thrdStarter(void *arg) * executing threads. It is added at the end of the list. * rgerhards, 2007-12-14 */ -rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *)) +rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), sbool bNeedsCancel) { DEFiRet; thrdInfo_t *pThis; @@ -159,6 +219,7 @@ rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdI pThis->bIsActive = 1; pThis->pUsrThrdMain = thrdMain; pThis->pAfterRun = afterRun; + pThis->bNeedsCancel = bNeedsCancel; i = pthread_create(&pThis->thrdID, NULL, thrdStarter, pThis); CHKiRet(llAppend(&llThrds, NULL, pThis)); @@ -184,37 +245,10 @@ rsRetVal thrdInit(void) rsRetVal thrdExit(void) { DEFiRet; - iRet = llDestroy(&llThrds); - - RETiRet; -} - - -/* thrdSleep() - a fairly portable way to put a thread to sleep. It - * will wake up when - * a) the wake-time is over - * b) the thread shall be terminated - * Returns RS_RET_OK if all went well, RS_RET_TERMINATE_NOW if the calling - * thread shall be terminated and any other state if an error happened. - * rgerhards, 2007-12-17 - */ -rsRetVal -thrdSleep(thrdInfo_t *pThis, int iSeconds, int iuSeconds) -{ - DEFiRet; - struct timeval tvSelectTimeout; - - assert(pThis != NULL); - tvSelectTimeout.tv_sec = iSeconds; - tvSelectTimeout.tv_usec = iuSeconds; /* micro seconds */ - select(1, NULL, NULL, NULL, &tvSelectTimeout); - if(pThis->bShallStop) - iRet = RS_RET_TERMINATE_NOW; RETiRet; } -/* - * vi:set ai: +/* vi:set ai: */ @@ -25,12 +25,14 @@ /* the thread object */ struct thrdInfo { - pthread_mutex_t *mutTermOK; /* Is it ok to terminate that thread now? */ + pthread_mutex_t mutThrd;/* mutex for handling long-running operations and shutdown */ + pthread_cond_t condThrdTerm;/* condition: thread terminates (used just for shutdown loop) */ int bIsActive; /* Is thread running? */ int bShallStop; /* set to 1 if the thread should be stopped ? */ rsRetVal (*pUsrThrdMain)(struct thrdInfo*); /* user thread main to be called in new thread */ rsRetVal (*pAfterRun)(struct thrdInfo*); /* cleanup function */ pthread_t thrdID; + sbool bNeedsCancel; /* must input be terminated by pthread_cancel()? */ }; /* prototypes */ @@ -38,8 +40,7 @@ rsRetVal thrdExit(void); rsRetVal thrdInit(void); rsRetVal thrdTerminate(thrdInfo_t *pThis); rsRetVal thrdTerminateAll(void); -rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *)); -rsRetVal thrdSleep(thrdInfo_t *pThis, int iSeconds, int iuSeconds); +rsRetVal thrdCreate(rsRetVal (*thrdMain)(thrdInfo_t*), rsRetVal(*afterRun)(thrdInfo_t *), sbool); /* macros (replace inline functions) */ diff --git a/tools/Makefile.am b/tools/Makefile.am index 1497d3be..8f2989ca 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -17,6 +17,18 @@ rsyslogd_SOURCES = \ ompipe.h \ omdiscard.c \ omdiscard.h \ + pmrfc5424.c \ + pmrfc5424.h \ + pmrfc3164.c \ + pmrfc3164.h \ + smtradfile.c \ + smtradfile.h \ + smfile.c \ + smfile.h \ + smfwd.c \ + smfwd.h \ + smtradfwd.c \ + smtradfwd.h \ iminternal.c \ iminternal.h \ pidfile.c \ diff --git a/tools/iminternal.c b/tools/iminternal.c index 0ceff3d8..bd1fa128 100644 --- a/tools/iminternal.c +++ b/tools/iminternal.c @@ -89,7 +89,7 @@ finalize_it: * The interface of this function is modelled after syslogd/logmsg(), * for which it is an "replacement". */ -rsRetVal iminternalAddMsg(int pri, msg_t *pMsg, int flags) +rsRetVal iminternalAddMsg(int pri, msg_t *pMsg) { DEFiRet; iminternal_t *pThis; @@ -100,7 +100,6 @@ rsRetVal iminternalAddMsg(int pri, msg_t *pMsg, int flags) pThis->pri = pri; pThis->pMsg = pMsg; - pThis->flags = flags; CHKiRet(llAppend(&llMsgs, NULL, (void*) pThis)); @@ -119,7 +118,7 @@ finalize_it: * from the list and return it to the caller. The caller is * responsible for freeing the message! */ -rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg, int *pFlags) +rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg) { DEFiRet; iminternal_t *pThis; @@ -127,11 +126,9 @@ rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg, int *pFlags) assert(pPri != NULL); assert(ppMsg != NULL); - assert(pFlags != NULL); CHKiRet(llGetNextElt(&llMsgs, &llCookie, (void*)&pThis)); *pPri = pThis->pri; - *pFlags = pThis->flags; *ppMsg = pThis->pMsg; pThis->pMsg = NULL; /* we do no longer own it - important for destructor */ diff --git a/tools/iminternal.h b/tools/iminternal.h index 8dc0f171..f1062a15 100644 --- a/tools/iminternal.h +++ b/tools/iminternal.h @@ -35,15 +35,14 @@ struct iminternal_s { /* config file sysline parse entry */ int pri; msg_t *pMsg; /* the message (in all its glory) */ - int flags; }; typedef struct iminternal_s iminternal_t; /* prototypes */ rsRetVal modInitIminternal(void); rsRetVal modExitIminternal(void); -rsRetVal iminternalAddMsg(int pri, msg_t *pMsg, int flags); +rsRetVal iminternalAddMsg(int pri, msg_t *pMsg); rsRetVal iminternalHaveMsgReady(int* pbHaveOne); -rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg, int *pFlags); +rsRetVal iminternalRemoveMsg(int *pPri, msg_t **ppMsg); #endif /* #ifndef IMINTERNAL_H_INCLUDED */ diff --git a/tools/omfile.c b/tools/omfile.c index a45d904b..57089cfd 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -48,10 +48,13 @@ #include <libgen.h> #include <unistd.h> #include <sys/file.h> - #ifdef OS_SOLARIS # include <fcntl.h> #endif +#ifdef HAVE_ATOMIC_BUILTINS +# include <pthread.h> +#endif + #include "conf.h" #include "syslogd-types.h" @@ -74,12 +77,37 @@ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(strm) +/* for our current LRU mechanism, we need a monotonically increasing counters. We use + * it much like a "Lamport logical clock": we do not need the actual time, we just need + * to know the sequence in which files were accessed. So we use a simple counter to + * create that sequence. We use an unsigned 64 bit value which is extremely unlike to + * wrap within the lifetime of a process. If we process 1,000,000 file writes per + * second, the process could still exist over 500,000 years before a wrap to 0 happens. + * That should be sufficient (and even than, there would no really bad effect ;)). + * The variable below is the global counter/clock. + */ +#if HAVE_ATOMIC_BUILTINS_64BIT +static uint64 clockFileAccess = 0; +#else +static unsigned clockFileAccess = 0; +#endif +/* and the "tick" function */ +#ifndef HAVE_ATOMIC_BUILTINS +static pthread_mutex_t mutClock; +#endif +static inline uint64 +getClockFileAccess(void) +{ + return ATOMIC_INC_AND_FETCH(&clockFileAccess, &mutClock); +} + + /* The following structure is a dynafile name cache entry. */ struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ strm_t *pStrm; /* our output stream */ - time_t lastUsed; /* for LRU - last access */ + uint64 clkTickAccessed;/* for LRU - based on clockFileAccess */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; @@ -103,7 +131,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ -static bool bFlushOnTXEnd = FLUSHONTX_DFLT;/* flush write buffers when transaction has ended? */ +static sbool bFlushOnTXEnd = FLUSHONTX_DFLT;/* flush write buffers when transaction has ended? */ static int64 iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static int bUseAsyncWriter = USE_ASYNCWRITER_DFLT; /* should we enable asynchronous writing? */ @@ -120,7 +148,7 @@ typedef struct _instanceData { int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ - bool bForceChown; /* force chown() on existing files? */ + sbool bForceChown; /* force chown() on existing files? */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -139,8 +167,8 @@ typedef struct _instanceData { int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ int iFlushInterval; /* how fast flush buffer on inactivity? */ - bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ - bool bUseAsyncWriter; /* use async stream writer? */ + sbool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ + sbool bUseAsyncWriter; /* use async stream writer? */ } instanceData; @@ -456,7 +484,7 @@ finalize_it: static inline rsRetVal prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) { - time_t ttOldest; /* timestamp of oldest element */ + uint64 ctOldest; /* "timestamp" of oldest element */ int iOldest; int i; int iFirstFree; @@ -475,7 +503,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) if( (pData->iCurrElt != -1) && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ - pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ // TODO: optimize time call! + pCache[pData->iCurrElt]->clkTickAccessed = getClockFileAccess(); // LRU needs only a strictly monotonically increasing counter, so such a one could do FINALIZE; } @@ -486,7 +514,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) pData->iCurrElt = -1; /* invalid current element pointer */ iFirstFree = -1; /* not yet found */ iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */ - ttOldest = time(NULL) + 1; /* there must always be an older one */ + ctOldest = getClockFileAccess(); /* there must always be an older one */ for(i = 0 ; i < pData->iCurrCacheSize ; ++i) { if(pCache[i] == NULL || pCache[i]->pName == NULL) { if(iFirstFree == -1) @@ -496,12 +524,12 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) /* we found our element! */ pData->pStrm = pCache[i]->pStrm; pData->iCurrElt = i; - pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */ + pCache[i]->clkTickAccessed = getClockFileAccess(); /* update "timestamp" for LRU */ FINALIZE; } /* did not find it - so lets keep track of the counters for LRU */ - if(pCache[i]->lastUsed < ttOldest) { - ttOldest = pCache[i]->lastUsed; + if(pCache[i]->clkTickAccessed < ctOldest) { + ctOldest = pCache[i]->clkTickAccessed; iOldest = i; } } @@ -527,7 +555,6 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) iFirstFree = pData->iCurrCacheSize++; } -// RG: this is the begin of a potential problem area /* Note that the following code sequence does not work with the cache entry itself, * but rather with pData->pStrm, the (sole) stream pointer in the non-dynafile case. * The cache array is only updated after the open was successful. -- rgerhards, 2010-03-21 @@ -565,7 +592,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } pCache[iFirstFree]->pStrm = pData->pStrm; - pCache[iFirstFree]->lastUsed = time(NULL); + pCache[iFirstFree]->clkTickAccessed = getClockFileAccess(); pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); @@ -586,7 +613,7 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pData != NULL); ASSERT(pszBuf != NULL); -dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); +dbgprintf("write to stream, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); FINALIZE; @@ -652,15 +679,31 @@ BEGINtryResume CODESTARTtryResume ENDtryResume +BEGINbeginTransaction +CODESTARTbeginTransaction + /* we have nothing to do to begin a transaction */ +ENDbeginTransaction + + +BEGINendTransaction +CODESTARTendTransaction + if(pData->bFlushOnTXEnd) { + CHKiRet(strm.Flush(pData->pStrm)); + } +finalize_it: +ENDendTransaction + + BEGINdoAction CODESTARTdoAction DBGPRINTF("file to log to: %s\n", pData->f_fname); CHKiRet(writeFile(ppString, iMsgOpts, pData)); - if(pData->bFlushOnTXEnd) { - /* TODO v5: do this in endTransaction only! */ + if(!bCoreSupportsBatching && pData->bFlushOnTXEnd) { CHKiRet(strm.Flush(pData->pStrm)); } finalize_it: + if(iRet == RS_RET_OK) + iRet = RS_RET_DEFER_COMMIT; ENDdoAction @@ -817,12 +860,14 @@ CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); objRelease(strm, CORE_COMPONENT); free(pszFileDfltTplName); + DESTROY_ATOMIC_HELPER_MUT(mutClock); ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ CODEqueryEtryPt_doHUP ENDqueryEtryPt @@ -833,6 +878,11 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(strm, CORE_COMPONENT)); + + INIT_ATOMIC_HELPER_MUT(mutClock); + + INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); + DBGPRINTF("omfile: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not "); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); diff --git a/tools/omfwd.c b/tools/omfwd.c index cbfc36a4..481fabab 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -10,7 +10,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -436,13 +436,13 @@ CODESTARTdoAction * hard-coded but this may be changed to a config parameter. * rgerhards, 2006-11-30 */ - if(pData->compressionLevel && (l > MIN_SIZE_FOR_COMPRESS)) { + if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { Bytef *out; uLongf destLen = iMaxLine + iMaxLine/100 +12; /* recommended value from zlib doc */ uLong srcLen = l; int ret; /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ - CHKmalloc(out = (Bytef*) malloc(destLen)); + CHKmalloc(out = (Bytef*) MALLOC(destLen)); out[0] = 'z'; out[1] = '\0'; ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, @@ -626,7 +626,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) tmp = ++p; for(i=0 ; *p && isdigit((int) *p) ; ++p, ++i) /* SKIP AND COUNT */; - pData->port = malloc(i + 1); + pData->port = MALLOC(i + 1); if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store syslog forwarding port, " "using default port, results may not be what you intend\n"); diff --git a/tools/ompipe.c b/tools/ompipe.c index 2d621e0e..cf22bc84 100644 --- a/tools/ompipe.c +++ b/tools/ompipe.c @@ -38,6 +38,7 @@ #include <stdlib.h> #include <unistd.h> #include <string.h> +#include <unistd.h> #include <assert.h> #include <errno.h> #include <fcntl.h> diff --git a/tools/pmrfc3164.c b/tools/pmrfc3164.c new file mode 100644 index 00000000..38f556a2 --- /dev/null +++ b/tools/pmrfc3164.c @@ -0,0 +1,240 @@ +/* pmrfc3164.c + * This is a parser module for RFC3164(legacy syslog)-formatted messages. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2009-11-04 by RGerhards + * + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include <ctype.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.rfc3164") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +/* parse a legay-formatted syslog message. + */ +BEGINparse + uchar *p2parse; + int lenMsg; + int bTAGCharDetected; + int i; /* general index for parsing */ + uchar bufParseTAG[CONF_TAG_MAXSIZE]; + uchar bufParseHOSTNAME[CONF_HOSTNAME_MAXSIZE]; +CODESTARTparse + dbgprintf("Message will now be parsed by the legacy syslog parser (one size fits all... ;)).\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + setProtocolVersion(pMsg, 0); + + /* Check to see if msg contains a timestamp. We start by assuming + * that the message timestamp is the time of reception (which we + * generated ourselfs and then try to actually find one inside the + * message. There we go from high-to low precison and are done + * when we find a matching one. -- rgerhards, 2008-09-16 + */ + if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* we are done - parse pointer is moved by ParseTIMESTAMP3339 */; + } else if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; + } else if(*p2parse == ' ' && lenMsg > 1) { /* try to see if it is slighly malformed - HP procurve seems to do that sometimes */ + ++p2parse; /* move over space */ + --lenMsg; + if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + /* indeed, we got it! */ + /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; + } else {/* parse pointer needs to be restored, as we moved it off-by-one + * for this try. + */ + --p2parse; + ++lenMsg; + } + } + + if(pMsg->msgFlags & IGNDATE) { + /* we need to ignore the msg data, so simply copy over reception date */ + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + } + + /* rgerhards, 2006-03-13: next, we parse the hostname and tag. But we + * do this only when the user has not forbidden this. I now introduce some + * code that allows a user to configure rsyslogd to treat the rest of the + * message as MSG part completely. In this case, the hostname will be the + * machine that we received the message from and the tag will be empty. This + * is meant to be an interim solution, but for now it is in the code. + */ + if(bParseHOSTNAMEandTAG && !(pMsg->msgFlags & INTERNAL_MSG)) { + /* parse HOSTNAME - but only if this is network-received! + * rger, 2005-11-14: we still have a problem with BSD messages. These messages + * do NOT include a host name. In most cases, this leads to the TAG to be treated + * as hostname and the first word of the message as the TAG. Clearly, this is not + * of advantage ;) I think I have now found a way to handle this situation: there + * are certain characters which are frequently used in TAG (e.g. ':'), which are + * *invalid* in host names. So while parsing the hostname, I check for these characters. + * If I find them, I set a simple flag but continue. After parsing, I check the flag. + * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change + * the fields. I think this logic shall work with any type of syslog message. + * rgerhards, 2009-06-23: and I now have extended this logic to every character + * that is not a valid hostname. + */ + bTAGCharDetected = 0; + if(lenMsg > 0 && pMsg->msgFlags & PARSE_HOSTNAME) { + i = 0; + while(i < lenMsg && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.' + || p2parse[i] == '_' || p2parse[i] == '-') && i < (CONF_HOSTNAME_MAXSIZE - 1)) { + bufParseHOSTNAME[i] = p2parse[i]; + ++i; + } + + if(i == lenMsg) { + /* we have a message that is empty immediately after the hostname, + * but the hostname thus is valid! -- rgerhards, 2010-02-22 + */ + p2parse += i; + lenMsg -= i; + bufParseHOSTNAME[i] = '\0'; + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); + } else if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) { + /* we got a hostname! */ + p2parse += i + 1; /* "eat" it (including SP delimiter) */ + lenMsg -= i + 1; + bufParseHOSTNAME[i] = '\0'; + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); + } + } + + /* now parse TAG - that should be present in message from all sources. + * This code is somewhat not compliant with RFC 3164. As of 3164, + * the TAG field is ended by any non-alphanumeric character. In + * practice, however, the TAG often contains dashes and other things, + * which would end the TAG. So it is not desirable. As such, we only + * accept colon and SP to be terminators. Even there is a slight difference: + * a colon is PART of the TAG, while a SP is NOT part of the tag + * (it is CONTENT). Starting 2008-04-04, we have removed the 32 character + * size limit (from RFC3164) on the tag. This had bad effects on existing + * envrionments, as sysklogd didn't obey it either (probably another bug + * in RFC3164...). We now receive the full size, but will modify the + * outputs so that only 32 characters max are used by default. + */ + i = 0; + while(lenMsg > 0 && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *p2parse++; + --lenMsg; + } + if(lenMsg > 0 && *p2parse == ':') { + ++p2parse; + --lenMsg; + bufParseTAG[i++] = ':'; + } + + /* no TAG can only be detected if the message immediatly ends, in which case an empty TAG + * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23 + */ + bufParseTAG[i] = '\0'; /* terminate string */ + MsgSetTAG(pMsg, bufParseTAG, i); + } else {/* we enter this code area when the user has instructed rsyslog NOT + * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 + */ + if(!(pMsg->msgFlags & INTERNAL_MSG)) { + DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); + } + } + + /* The rest is the actual MSG */ + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(pmrfc3164) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("rfc3164 parser init called\n"); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ diff --git a/tools/pmrfc3164.h b/tools/pmrfc3164.h new file mode 100644 index 00000000..24304094 --- /dev/null +++ b/tools/pmrfc3164.h @@ -0,0 +1,33 @@ +/* pmrfc3164.h + * These are the definitions for the RFC3164 parser module. + * + * File begun on 2009-11-04 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef PMRFC3164_H_INCLUDED +#define PMRFC3164_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitpmrfc3164(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef PMRFC3164_H_INCLUDED */ +/* vi:set ai: + */ diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c new file mode 100644 index 00000000..07994ade --- /dev/null +++ b/tools/pmrfc5424.c @@ -0,0 +1,329 @@ +/* pmrfc5424.c + * This is a parser module for RFC5424-formatted messages. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2009-11-03 by RGerhards + * + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.rfc5424") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* config data */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +/* Helper to parseRFCSyslogMsg. This function parses a field up to + * (and including) the SP character after it. The field contents is + * returned in a caller-provided buffer. The parsepointer is advanced + * to after the terminating SP. The caller must ensure that the + * provided buffer is large enough to hold the to be extracted value. + * Returns 0 if everything is fine or 1 if either the field is not + * SP-terminated or any other error occurs. -- rger, 2005-11-24 + * The function now receives the size of the string and makes sure + * that it does not process more than that. The *pLenStr counter is + * updated on exit. -- rgerhards, 2009-09-23 + */ +static int parseRFCField(uchar **pp2parse, uchar *pResult, int *pLenStr) +{ + uchar *p2parse; + int iRet = 0; + + assert(pp2parse != NULL); + assert(*pp2parse != NULL); + assert(pResult != NULL); + + p2parse = *pp2parse; + + /* this is the actual parsing loop */ + while(*pLenStr > 0 && *p2parse != ' ') { + *pResult++ = *p2parse++; + --(*pLenStr); + } + + if(*pLenStr > 0 && *p2parse == ' ') { + ++p2parse; /* eat SP, but only if not at end of string */ + --(*pLenStr); + } else { + iRet = 1; /* there MUST be an SP! */ + } + *pResult = '\0'; + + /* set the new parse pointer */ + *pp2parse = p2parse; + return 0; +} + + +/* Helper to parseRFCSyslogMsg. This function parses the structured + * data field of a message. It does NOT parse inside structured data, + * just gets the field as whole. Parsing the single entities is left + * to other functions. The parsepointer is advanced + * to after the terminating SP. The caller must ensure that the + * provided buffer is large enough to hold the to be extracted value. + * Returns 0 if everything is fine or 1 if either the field is not + * SP-terminated or any other error occurs. -- rger, 2005-11-24 + * The function now receives the size of the string and makes sure + * that it does not process more than that. The *pLenStr counter is + * updated on exit. -- rgerhards, 2009-09-23 + */ +static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr) +{ + uchar *p2parse; + int bCont = 1; + int iRet = 0; + int lenStr; + + assert(pp2parse != NULL); + assert(*pp2parse != NULL); + assert(pResult != NULL); + + p2parse = *pp2parse; + lenStr = *pLenStr; + + /* this is the actual parsing loop + * Remeber: structured data starts with [ and includes any characters + * until the first ] followed by a SP. There may be spaces inside + * structured data. There may also be \] inside the structured data, which + * do NOT terminate an element. + */ + if(lenStr == 0 || *p2parse != '[') + return 1; /* this is NOT structured data! */ + + if(*p2parse == '-') { /* empty structured data? */ + *pResult++ = '-'; + ++p2parse; + --lenStr; + } else { + while(bCont) { + if(lenStr < 2) { + /* we now need to check if we have only structured data */ + if(lenStr > 0 && *p2parse == ']') { + *pResult++ = *p2parse; + p2parse++; + lenStr--; + bCont = 0; + } else { + iRet = 1; /* this is not valid! */ + bCont = 0; + } + } else if(*p2parse == '\\' && *(p2parse+1) == ']') { + /* this is escaped, need to copy both */ + *pResult++ = *p2parse++; + *pResult++ = *p2parse++; + lenStr -= 2; + } else if(*p2parse == ']' && *(p2parse+1) == ' ') { + /* found end, just need to copy the ] and eat the SP */ + *pResult++ = *p2parse; + p2parse += 2; + lenStr -= 2; + bCont = 0; + } else { + *pResult++ = *p2parse++; + --lenStr; + } + } + } + + if(lenStr > 0 && *p2parse == ' ') { + ++p2parse; /* eat SP, but only if not at end of string */ + --lenStr; + } else { + iRet = 1; /* there MUST be an SP! */ + } + *pResult = '\0'; + + /* set the new parse pointer */ + *pp2parse = p2parse; + *pLenStr = lenStr; + return 0; +} + +/* parse a RFC5424-formatted syslog message. This function returns + * 0 if processing of the message shall continue and 1 if something + * went wrong and this messe should be ignored. This function has been + * implemented in the effort to support syslog-protocol. Please note that + * the name (parse *RFC*) stems from the hope that syslog-protocol will + * some time become an RFC. Do not confuse this with informational + * RFC 3164 (which is legacy syslog). + * + * currently supported format: + * + * <PRI>VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP [SD-ID]s SP MSG + * + * <PRI> is already stripped when this function is entered. VERSION already + * has been confirmed to be "1", but has NOT been stripped from the message. + * + * rger, 2005-11-24 + */ +//static int parseRFCSyslogMsg(msg_t *pMsg, int flags) +BEGINparse + uchar *p2parse; + uchar *pBuf = NULL; + int lenMsg; + int bContParse = 1; +CODESTARTparse + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; + + /* check if we are the right parser */ + if(lenMsg < 2 || p2parse[0] != '1' || p2parse[1] != ' ') { + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + DBGPRINTF("Message has RFC5424/syslog-protocol format.\n"); + setProtocolVersion(pMsg, 1); + p2parse += 2; + lenMsg -= 2; + + /* Now get us some memory we can use as a work buffer while parsing. + * We simply allocated a buffer sufficiently large to hold all of the + * message, so we can not run into any troubles. I think this is + * wiser than to use individual buffers. + */ + CHKmalloc(pBuf = MALLOC(sizeof(uchar) * (lenMsg + 1))); + + /* IMPORTANT NOTE: + * Validation is not actually done below nor are any errors handled. I have + * NOT included this for the current proof of concept. However, it is strongly + * advisable to add it when this code actually goes into production. + * rgerhards, 2005-11-24 + */ + + /* TIMESTAMP */ + if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + if(pMsg->msgFlags & IGNDATE) { + /* we need to ignore the msg data, so simply copy over reception date */ + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + } + } else { + DBGPRINTF("no TIMESTAMP detected!\n"); + bContParse = 0; + } + + /* HOSTNAME */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetHOSTNAME(pMsg, pBuf, ustrlen(pBuf)); + } + + /* APP-NAME */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetAPPNAME(pMsg, (char*)pBuf); + } + + /* PROCID */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetPROCID(pMsg, (char*)pBuf); + } + + /* MSGID */ + if(bContParse) { + parseRFCField(&p2parse, pBuf, &lenMsg); + MsgSetMSGID(pMsg, (char*)pBuf); + } + + /* STRUCTURED-DATA */ + if(bContParse) { + parseRFCStructuredData(&p2parse, pBuf, &lenMsg); + MsgSetStructuredData(pMsg, (char*)pBuf); + } + + /* MSG */ + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); + +finalize_it: + if(pBuf != NULL) + free(pBuf); +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(pmrfc5424) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("rfc5424 parser init called\n"); + dbgprintf("GetParserName addr %p\n", GetParserName); +ENDmodInit + +/* vim:set ai: + */ diff --git a/tools/pmrfc5424.h b/tools/pmrfc5424.h new file mode 100644 index 00000000..df2a1c81 --- /dev/null +++ b/tools/pmrfc5424.h @@ -0,0 +1,33 @@ +/* pmrfc5424.h + * These are the definitions for the RFCC5424 parser module. + * + * File begun on 2009-11-03 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef PMRFC54254_H_INCLUDED +#define PMRFC54254_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitpmrfc5424(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef PMRFC54254_H_INCLUDED */ +/* vi:set ai: + */ diff --git a/tools/smfile.c b/tools/smfile.c new file mode 100644 index 00000000..5e4a775f --- /dev/null +++ b/tools/smfile.c @@ -0,0 +1,135 @@ +/* smfile.c + * This is a strgen module for the traditional file format. + * + * Format generated: + * "%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" + * Note that this is the same as smtradfile.c, except that we do have a RFC3339 timestamp. However, + * we have copied over the code from there, it is too simple to go through all the hassle + * of having a single code base. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2010-06-01 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" + +MODULE_TYPE_STRGEN +STRGEN_NAME("RSYSLOG_FileFormat") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA + + +/* config data */ + + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + */ +BEGINstrgen + register int iBuf; + uchar *pTimeStamp; + size_t lenTimeStamp; + uchar *pHOSTNAME; + size_t lenHOSTNAME; + uchar *pTAG; + int lenTAG; + uchar *pMSG; + size_t lenMSG; + size_t lenTotal; +CODESTARTstrgen + /* first obtain all strings and their length (if not fixed) */ + pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); + lenTimeStamp = ustrlen(pTimeStamp); + pHOSTNAME = (uchar*) getHOSTNAME(pMsg); + lenHOSTNAME = getHOSTNAMELen(pMsg); + getTAG(pMsg, &pTAG, &lenTAG); + pMSG = getMSG(pMsg); + lenMSG = getMSGLen(pMsg); + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2; + if(pMSG[0] != ' ') + ++lenTotal; /* then we need to introduce one additional space */ + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + memcpy(*ppBuf, pTimeStamp, lenTimeStamp); + iBuf = lenTimeStamp; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME); + iBuf += lenHOSTNAME; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pTAG, lenTAG); + iBuf += lenTAG; + + if(pMSG[0] != ' ') + *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf + iBuf, pMSG, lenMSG); + iBuf += lenMSG; + + /* trailer */ + *(*ppBuf + iBuf++) = '\n'; + *(*ppBuf + iBuf) = '\0'; + +finalize_it: +ENDstrgen + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(smfile) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + + dbgprintf("rsyslog standard file format strgen init called, compiled with version %s\n", VERSION); +ENDmodInit diff --git a/tools/smfile.h b/tools/smfile.h new file mode 100644 index 00000000..10946db5 --- /dev/null +++ b/tools/smfile.h @@ -0,0 +1,31 @@ +/* smfile.h + * These are the definitions for the traditional file format stringen module. + * + * File begun on 2010-06-04 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef SMFILE_H_INCLUDED +#define SMFILE_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitsmfile(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef SMFILE_H_INCLUDED */ diff --git a/tools/smfwd.c b/tools/smfwd.c new file mode 100644 index 00000000..fe33fb2c --- /dev/null +++ b/tools/smfwd.c @@ -0,0 +1,142 @@ +/* smfwd.c + * This is a strgen module for the traditional (network) forwarding format. + * + * Format generated: + * "<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2010-06-01 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" + +MODULE_TYPE_STRGEN +STRGEN_NAME("RSYSLOG_ForwardFormat") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA + + +/* config data */ + + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + */ +BEGINstrgen + register int iBuf; + char *pPRI; + size_t lenPRI; + uchar *pTimeStamp; + size_t lenTimeStamp; + uchar *pHOSTNAME; + size_t lenHOSTNAME; + uchar *pTAG; + int lenTAG; + uchar *pMSG; + size_t lenMSG; + size_t lenTotal; +CODESTARTstrgen + /* first obtain all strings and their length (if not fixed) */ + pPRI = getPRI(pMsg); + lenPRI = strlen(pPRI); + pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date); + lenTimeStamp = ustrlen(pTimeStamp); + pHOSTNAME = (uchar*) getHOSTNAME(pMsg); + lenHOSTNAME = getHOSTNAMELen(pMsg); + getTAG(pMsg, &pTAG, &lenTAG); + if(lenTAG > 32) + lenTAG = 32; /* for forwarding, a max of 32 chars is permitted (RFC!) */ + pMSG = getMSG(pMsg); + lenMSG = getMSGLen(pMsg); + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = 1 + lenPRI + 1 + lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 1; + if(pMSG[0] != ' ') + ++lenTotal; /* then we need to introduce one additional space */ + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + **ppBuf = '<'; + memcpy(*ppBuf + 1, pPRI, lenPRI); + iBuf = lenPRI + 1; + *(*ppBuf + iBuf++) = '>'; + + memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp); + iBuf += lenTimeStamp; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME); + iBuf += lenHOSTNAME; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pTAG, lenTAG); + iBuf += lenTAG; + + if(pMSG[0] != ' ') + *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf + iBuf, pMSG, lenMSG); + iBuf += lenMSG; + + /* string terminator */ + *(*ppBuf + iBuf) = '\0'; + +finalize_it: +ENDstrgen + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(smfwd) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + + dbgprintf("rsyslog standard (network) forward format strgen init called, compiled with version %s\n", VERSION); +ENDmodInit diff --git a/tools/smfwd.h b/tools/smfwd.h new file mode 100644 index 00000000..191a6bf1 --- /dev/null +++ b/tools/smfwd.h @@ -0,0 +1,30 @@ +/* smfwd.h + * + * File begun on 2010-06-04 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef SMFWD_H_INCLUDED +#define SMFWD_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitsmfwd(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef SMFWD_H_INCLUDED */ diff --git a/tools/smtradfile.c b/tools/smtradfile.c new file mode 100644 index 00000000..eff2f99a --- /dev/null +++ b/tools/smtradfile.c @@ -0,0 +1,128 @@ +/* smtradfile.c + * This is a strgen module for the traditional file format. + * + * Format generated: + * "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2010-06-01 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" + +MODULE_TYPE_STRGEN +STRGEN_NAME("RSYSLOG_TraditionalFileFormat") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA + + +/* config data */ + + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + */ +BEGINstrgen + register int iBuf; + uchar *pTimeStamp; + uchar *pHOSTNAME; + size_t lenHOSTNAME; + uchar *pTAG; + int lenTAG; + uchar *pMSG; + size_t lenMSG; + size_t lenTotal; +CODESTARTstrgen + /* first obtain all strings and their length (if not fixed) */ + pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3164Date); + pHOSTNAME = (uchar*) getHOSTNAME(pMsg); + lenHOSTNAME = getHOSTNAMELen(pMsg); + getTAG(pMsg, &pTAG, &lenTAG); + pMSG = getMSG(pMsg); + lenMSG = getMSGLen(pMsg); + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = CONST_LEN_TIMESTAMP_3164 + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2; + if(pMSG[0] != ' ') + ++lenTotal; /* then we need to introduce one additional space */ + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + memcpy(*ppBuf, pTimeStamp, CONST_LEN_TIMESTAMP_3164); + *(*ppBuf + CONST_LEN_TIMESTAMP_3164) = ' '; + + memcpy(*ppBuf + CONST_LEN_TIMESTAMP_3164 + 1, pHOSTNAME, lenHOSTNAME); + iBuf = CONST_LEN_TIMESTAMP_3164 + 1 + lenHOSTNAME; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pTAG, lenTAG); + iBuf += lenTAG; + + if(pMSG[0] != ' ') + *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf + iBuf, pMSG, lenMSG); + iBuf += lenMSG; + + /* trailer */ + *(*ppBuf + iBuf++) = '\n'; + *(*ppBuf + iBuf) = '\0'; + +finalize_it: +ENDstrgen + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(smtradfile) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + dbgprintf("traditional file format strgen init called, compiled with version %s\n", VERSION); +ENDmodInit diff --git a/tools/smtradfile.h b/tools/smtradfile.h new file mode 100644 index 00000000..afc737ed --- /dev/null +++ b/tools/smtradfile.h @@ -0,0 +1,31 @@ +/* smtradfile.h + * These are the definitions for the traditional file format stringen module. + * + * File begun on 2010-06-01 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef SMTRADFILE_H_INCLUDED +#define SMTRADFILE_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitsmtradfile(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef SMTRADFILE_H_INCLUDED */ diff --git a/tools/smtradfwd.c b/tools/smtradfwd.c new file mode 100644 index 00000000..88dc6082 --- /dev/null +++ b/tools/smtradfwd.c @@ -0,0 +1,139 @@ +/* smtradfwd.c + * This is a strgen module for the traditional forwarding format. + * + * Format generated: + * "<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2010-06-01 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" + +MODULE_TYPE_STRGEN +STRGEN_NAME("RSYSLOG_TraditionalForwardFormat") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA + + +/* config data */ + + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + */ +BEGINstrgen + register int iBuf; + char *pPRI; + size_t lenPRI; + uchar *pTimeStamp; + uchar *pHOSTNAME; + size_t lenHOSTNAME; + uchar *pTAG; + int lenTAG; + uchar *pMSG; + size_t lenMSG; + size_t lenTotal; +CODESTARTstrgen + /* first obtain all strings and their length (if not fixed) */ + pPRI = getPRI(pMsg); + lenPRI = strlen(pPRI); + pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3164Date); + pHOSTNAME = (uchar*) getHOSTNAME(pMsg); + lenHOSTNAME = getHOSTNAMELen(pMsg); + getTAG(pMsg, &pTAG, &lenTAG); + if(lenTAG > 32) + lenTAG = 32; /* for forwarding, a max of 32 chars is permitted (RFC!) */ + pMSG = getMSG(pMsg); + lenMSG = getMSGLen(pMsg); + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = 1 + lenPRI + 1 + CONST_LEN_TIMESTAMP_3164 + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 1; + if(pMSG[0] != ' ') + ++lenTotal; /* then we need to introduce one additional space */ + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + **ppBuf = '<'; + memcpy(*ppBuf + 1, pPRI, lenPRI); + iBuf = lenPRI + 1; + *(*ppBuf + iBuf++) = '>'; + + memcpy(*ppBuf + iBuf, pTimeStamp, CONST_LEN_TIMESTAMP_3164); + iBuf += CONST_LEN_TIMESTAMP_3164; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME); + iBuf += lenHOSTNAME; + *(*ppBuf + iBuf++) = ' '; + + memcpy(*ppBuf + iBuf, pTAG, lenTAG); + iBuf += lenTAG; + + if(pMSG[0] != ' ') + *(*ppBuf + iBuf++) = ' '; + memcpy(*ppBuf + iBuf, pMSG, lenMSG); + iBuf += lenMSG; + + /* string terminator */ + *(*ppBuf + iBuf) = '\0'; + +finalize_it: +ENDstrgen + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit(smtradfwd) +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + dbgprintf("rsyslog traditional (network) forward format strgen init called, compiled with version %s\n", VERSION); +ENDmodInit diff --git a/tools/smtradfwd.h b/tools/smtradfwd.h new file mode 100644 index 00000000..9ff0ab54 --- /dev/null +++ b/tools/smtradfwd.h @@ -0,0 +1,30 @@ +/* smtradfwd.h + * + * File begun on 2010-06-04 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#ifndef SMTRADFWD_H_INCLUDED +#define SMTRADFWD_H_INCLUDED 1 + +/* prototypes */ +rsRetVal modInitsmtradfwd(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*); + +#endif /* #ifndef SMTRADFWD_H_INCLUDED */ diff --git a/tools/syncdemo.c b/tools/syncdemo.c new file mode 100644 index 00000000..89a5c6cc --- /dev/null +++ b/tools/syncdemo.c @@ -0,0 +1,440 @@ +/* syncdemo - a program to demonstrate the performance and validity of different + * synchronization methods as well as some timing properties. + * + * The task to be done is very simple: a single gloabl integer is to to incremented + * by multiple threads. All this is done in a very-high concurrency environment. Note that + * the test is unfair to mechanisms likes spinlocks, because we have almost only wait + * time but no real processing time between the waits. However, the test provides + * some good insight into atomic instructions vs. other synchronisation methods. + * It also proves that garbling variables by not doing proper synchronisation is + * highly likely. For best results, this program should be executed on a + * multiprocessor machine (on a uniprocessor, it will probably not display the + * problems caused by missing synchronisation). + * + * Note: partitioned processing mode means that all computation is first done + * locally and the final result is then combined doing proper synchronization. + * This mode is used as a baseline for uninterrupted processing. + * + * compile with $ gcc -O1 -o syncdemo -lpthread syncdemo.c + * + * Alternatively, you may use -O0, but not a higher level. Note that + * the gcc code generator does in neither case generate code really + * suitable to compare "part" and "none" modes. If you absolutely need + * to do that, you need to use inline assembly. However, the results should + * be fairly OK when consitently using either -O0 or -O1. If you see a big loss + * of performance when you compare "none" and "part", be sure to run + * "none" with -t1 and watch out for the results! In any case, looking at the generated + * assembly code is vital to interpret results correctly. Review of generated assembly + * done on 2010-05-05 indicates that -O0 is probably the best choice. Note that we + * use the volatile attribute in one spot. This is used because it results in the + * best comparable result for our gcc 4.4.3, not really to invoke the volatile semantics. + * + * use "gcc -g -Wa,-ahl=syncdemo.s -lpthread syncdemo.c" to obtain a mixed code/assembly listing. + * + * This program REQUIRES linux. With slight modification, it may run on Solaris. + * Note that gcc on Sparc does NOT offer atomic instruction support! + * + * Copyright (C) 2010 by Rainer Gerhards <rgerhards@hq.adiscon.com> + * Released under the GNU GPLv3. + * + * Inspired by (retrieved 2010-04-13) + * http://www.alexonlinux.com/multithreaded-simple-data-type-access-and-atomic-variables + */ +#define _GNU_SOURCE +#include <sched.h> +#include <stdio.h> +#include <pthread.h> +#include <unistd.h> +#include <semaphore.h> +#include <stdlib.h> +#include <linux/unistd.h> +#include <sys/syscall.h> +#include <sys/time.h> +#include <errno.h> +#include <getopt.h> + + +typedef enum { part, none, atomic, cas, spinlock, mutex, semaphore } syncType_t; +static syncType_t syncTypes[] = { part, none, atomic, cas, spinlock, mutex, semaphore }; + +/* config settings */ +static int bCPUAffinity = 0; +static int procs = 0; /* number of processors */ +static int numthrds = 0; /* if zero, => equal num of processors */ +static unsigned goal = 50000000; /* 50 million */ +static int bCSV = 0; /* generate CSV output? */ +static int numIterations = 1; /* number of iterations */ +static int dummyLoad = 0; /* number of dummy load iterations to generate */ +syncType_t syncType; +static int bAllSyncTypes = 0; + +static int global_int = 0; /* our global counter */ +static unsigned thrd_WorkToDo; /* number of computations each thread must do */ +static volatile int bStartRun = 0; /* indicate to flag when threads should start */ + +static struct timeval tvStart, tvEnd; /* used for timing one testing iteration */ + +/* statistic counters */ +static long long totalRuntime; +static unsigned minRuntime = 999999999; +static unsigned maxRuntime = 0; + +/* sync objects (if needed) */ +static pthread_mutex_t mut; +static pthread_spinlock_t spin; +static sem_t sem; + +static char* +getSyncMethName(syncType_t st) +{ + switch(st) { + case part : return "partition"; + case none : return "none"; + case atomic : return "atomic op"; + case spinlock : return "spin lock"; + case mutex : return "mutex"; + case semaphore: return "semaphore"; + case cas : return "cas"; + } +} + + +static pid_t +gettid() +{ + return syscall( __NR_gettid ); +} + + +void *workerThread( void *arg ) +{ + int i, j; + volatile int partval = 0; /* use volatile so that gcc generates code similar to global var */ + int *partptr; + int oldval, newval; /* for CAS sync mode */ + int thrd_num = (int)(long)arg; + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(thrd_num % procs, &set); + if(syncType == part) { + partval = 0; + } + + /* if enabled, try to put thread on a fixed CPU (the one that corresponds to the + * thread ID). This may + */ + if(bCPUAffinity) { + if (sched_setaffinity( gettid(), sizeof( cpu_set_t ), &set )) { + perror( "sched_setaffinity" ); + return NULL; + } + } + + /* wait for "go" */ + while(bStartRun == 0) + /*WAIT!*/; + + for (i = 0; i < thrd_WorkToDo; i++) { + switch(syncType) { + case part: + ///* one needs to use inline assembly to get this right... */ + //asm("addl $1, global_int(%rip)"); + partval++; + break; + case none: + global_int++; + break; + case atomic: + __sync_fetch_and_add(&global_int,1); + break; + case cas: + do { + oldval = global_int; + newval = oldval + 1; + } while(!__sync_bool_compare_and_swap(&global_int, oldval, newval)); + break; + case mutex: + pthread_mutex_lock(&mut); + global_int++; + pthread_mutex_unlock(&mut); + break; + case spinlock: + pthread_spin_lock(&spin); + global_int++; + pthread_spin_unlock(&spin); + break; + case semaphore: + sem_wait(&sem); + global_int++; + sem_post(&sem); + break; + } + + /* we now generate "dummy load" if instructed to do so. The idea is that + * we do some other work, as in real life, so that we have a better + * ratio of sync vs. actual work to do. + */ + for(j = 0 ; j < dummyLoad ; ++j) { + /* be careful: compiler may optimize loop out! */; + } + } + + if(syncType == part) { + pthread_mutex_lock(&mut); + global_int += partval; + pthread_mutex_unlock(&mut); + } + + return NULL; +} + + +static void beginTiming(void) +{ + if(!(bCSV || bAllSyncTypes)) { + printf("Test Parameters:\n"); + printf("\tNumber of Cores.........: %d\n", procs); + printf("\tNumber of Threads.......: %d\n", numthrds); + printf("\tSet Affinity............: %s\n", bCPUAffinity ? "yes" : "no"); + printf("\tCount to................: %u\n", goal); + printf("\tWork for each Thread....: %u\n", thrd_WorkToDo); + printf("\tDummy Load Counter......: %d\n", dummyLoad); + printf("\tSync Method used........: %s\n", getSyncMethName(syncType)); + } + gettimeofday(&tvStart, NULL); +} + + +static void endTiming(void) +{ + unsigned delta; + long sec, usec; + long runtime; + + gettimeofday(&tvEnd, NULL); + if(tvStart.tv_usec > tvEnd.tv_usec) { + tvEnd.tv_sec--; + tvEnd.tv_usec += 1000000; + } + + sec = tvEnd.tv_sec - tvStart.tv_sec; + usec = tvEnd.tv_usec - tvStart.tv_usec; + + delta = thrd_WorkToDo * numthrds - global_int; + if(!bAllSyncTypes) { + if(bCSV) { + printf("%s,%d,%d,%d,%u,%u,%ld.%06.6ld\n", + getSyncMethName(syncType), procs, numthrds, bCPUAffinity, goal, delta, sec, usec); + } else { + printf("measured (sytem time) runtime is %ld.% 6.6ld seconds\n", sec, usec); + if(delta == 0) { + printf("Computation was done correctly.\n"); + } else { + printf("Computation INCORRECT,\n" + "\texpected %9u\n" + "\treal %9u\n" + "\toff by %9u\n", + thrd_WorkToDo * numthrds, + global_int, + delta); + } + } + } + + runtime = sec * 1000 + (usec / 1000); + totalRuntime += runtime; + if(runtime < minRuntime) + minRuntime = runtime; + if(runtime > maxRuntime) + maxRuntime = runtime; +} + + +static void +usage(void) +{ + fprintf(stderr, "Usage: syncdemo -a -c<num> -t<num>\n"); + fprintf(stderr, "\t-a set CPU affinity\n"); + fprintf(stderr, "\t-i number of iterations\n"); + fprintf(stderr, "\t-c<num> count to <num>\n"); + fprintf(stderr, "\t-d<num> dummy load, <num> iterations\n"); + fprintf(stderr, "\t-t<num> number of threads to use\n"); + fprintf(stderr, "\t-s<type> sync-type to use (none, atomic, mutex, spin, semaphore)\n"); + fprintf(stderr, "\t-C generate CSV output\n"); + fprintf(stderr, "\t-A test ALL sync types\n"); + exit(2); +} + + +/* carry out the actual test (one iteration) + */ +static void +singleTest(void) +{ + int i; + pthread_t *thrs; + + global_int = 0; + bStartRun = 0; + + thrs = malloc(sizeof(pthread_t) * numthrds); + if (thrs == NULL) { + perror( "malloc" ); + exit(1); + } + + thrd_WorkToDo = goal / numthrds; + + for (i = 0; i < numthrds; i++) { + if(pthread_create( &thrs[i], NULL, workerThread, (void *)(long)i )) { + perror( "pthread_create" ); + procs = i; + break; + } + } + + beginTiming(); + bStartRun = 1; /* start the threads (they are busy-waiting so far!) */ + + for (i = 0; i < numthrds; i++) + pthread_join( thrs[i], NULL ); + + endTiming(); + + free( thrs ); + +} + + +/* display an unsigned ms runtime count as string. Note that the + * string is inside a dynamically allocated buffer, which the caller + * must free to prevent a memory leak. + */ +char * +dispRuntime(unsigned rt) +{ + static char *fmtbuf; + + fmtbuf = malloc(32 * sizeof(char)); + snprintf(fmtbuf, 32, "%u.%03.3u", + rt / 1000, rt % 1000); + return(fmtbuf); +} + + +doTest(syncType_t st) +{ + int i; + + syncType = st; + totalRuntime = 0; + minRuntime = 999999999; + maxRuntime = 0; + for(i = 0 ; i < numIterations ; ++i) { + //printf("starting iteration %d\n", i); + singleTest(); + } + + /* we have a memory leak due to calling dispRuntime(), but we don't + * care as we terminate immediately. + */ + printf("%-10s: total runtime %6ld.%3.3u, avg %s, min %s, max %s\n", + getSyncMethName(st), + (long)totalRuntime/1000, (unsigned)(totalRuntime % 1000), + dispRuntime((unsigned) (totalRuntime / numIterations)), + dispRuntime(minRuntime), + dispRuntime(maxRuntime)); +} + + +int +main(int argc, char *argv[]) +{ + int i; + int opt; + + while((opt = getopt(argc, argv, "ac:d:i:t:s:CA")) != EOF) { + switch((char)opt) { + case 'A': + bAllSyncTypes = 1; + break; + case 'a': + bCPUAffinity = 1; + break; + case 'c': + goal = (unsigned) atol(optarg); + break; + case 'd': + dummyLoad = atoi(optarg); + break; + case 'i': + numIterations = atoi(optarg); + break; + case 't': + numthrds = atoi(optarg); + break; + case 'C': + bCSV = 1; + break; + case 's': + if(!strcmp(optarg, "none")) + syncType = none; + else if(!strcmp(optarg, "part")) + syncType = part; + else if(!strcmp(optarg, "atomic")) + syncType = atomic; + else if(!strcmp(optarg, "cas")) + syncType = cas; + else if(!strcmp(optarg, "mutex")) { + syncType = mutex; + pthread_mutex_init(&mut, NULL); + } else if(!strcmp(optarg, "spin")) { + syncType = spinlock; + } else if(!strcmp(optarg, "semaphore")) { + syncType = semaphore; + sem_init(&sem, 0, 1); + } else { + fprintf(stderr, "error: invalid sync mode '%s'\n", optarg); + usage(); + } + break; + default:usage(); + break; + } + } + + /* for simplicity, we init all sync helpers no matter if we need them */ + pthread_mutex_init(&mut, NULL); + pthread_spin_init(&spin, PTHREAD_PROCESS_PRIVATE); + sem_init(&sem, 0, 1); + + /* Getting number of CPUs */ + procs = (int)sysconf(_SC_NPROCESSORS_ONLN); + if(procs < 0) { + perror("sysconf"); + return -1; + } + + if(numthrds < 1) { + numthrds = procs; + } + + if(bAllSyncTypes) { + for(i = 0 ; i < sizeof(syncTypes) / sizeof(syncType_t) ; ++i) { + doTest(syncTypes[i]); + } + printf("Done running tests, result based on:\n"); + printf("\tNumber of Cores.........: %d\n", procs); + printf("\tNumber of Threads.......: %d\n", numthrds); + printf("\tSet CPU Affinity........: %s\n", bCPUAffinity ? "yes" : "no"); + printf("\tCount to................: %u\n", goal); + printf("\tWork for each Thread....: %u\n", thrd_WorkToDo); + printf("\tDummy Load Counter......: %d\n", dummyLoad); + printf("\tIterations..............: %d\n", numIterations); + } else { + doTest(syncType); + } + + return 0; +} diff --git a/tools/syslogd.c b/tools/syslogd.c index 985957a3..2e4ea5d5 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -16,27 +16,12 @@ * parts of the code have been rewritten. * * This Project was intiated and is maintained by - * Rainer Gerhards <rgerhards@hq.adiscon.com>. See - * AUTHORS to learn who helped make it become a reality. + * Rainer Gerhards <rgerhards@hq.adiscon.com>. * - * If you have questions about rsyslogd in general, please email - * info@adiscon.com. To learn more about rsyslogd, please visit - * http://www.rsyslog.com. - * - * \author Rainer Gerhards <rgerhards@adiscon.com> - * \date 2003-10-17 - * Some initial modifications on the sysklogd package to support - * liblogging. These have actually not yet been merged to the - * source you see currently (but they hopefully will) - * - * \date 2004-10-28 - * Restarted the modifications of sysklogd. This time, we - * focus on a simpler approach first. The initial goal is to - * provide MySQL database support (so that syslogd can log - * to the database). + * For further information, please see http://www.rsyslog.com * * rsyslog - An Enhanced syslogd Replacement. - * Copyright 2003-2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2003-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -129,13 +114,21 @@ #include "omfile.h" #include "ompipe.h" #include "omdiscard.h" +#include "pmrfc5424.h" +#include "pmrfc3164.h" +#include "smfile.h" +#include "smtradfile.h" +#include "smfwd.h" +#include "smtradfwd.h" #include "threads.h" +#include "wti.h" #include "queue.h" #include "stream.h" #include "conf.h" #include "errmsg.h" #include "datetime.h" #include "parser.h" +#include "batch.h" #include "unicode-helper.h" #include "ruleset.h" #include "rule.h" @@ -146,7 +139,7 @@ /* definitions for objects we access */ DEFobjCurrIf(obj) DEFobjCurrIf(glbl) -DEFobjCurrIf(datetime) +DEFobjCurrIf(datetime) /* TODO: make go away! */ DEFobjCurrIf(conf) DEFobjCurrIf(expr) DEFobjCurrIf(module) @@ -154,6 +147,7 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) DEFobjCurrIf(prop) +DEFobjCurrIf(parser) DEFobjCurrIf(net) /* TODO: make go away! */ @@ -161,18 +155,6 @@ DEFobjCurrIf(net) /* TODO: make go away! */ static rsRetVal GlobalClassExit(void); -#ifndef UTMP_FILE -#ifdef UTMP_FILENAME -#define UTMP_FILE UTMP_FILENAME -#else -#ifdef _PATH_UTMP -#define UTMP_FILE _PATH_UTMP -#else -#define UTMP_FILE "/etc/utmp" -#endif -#endif -#endif - #ifndef _PATH_LOGCONF #define _PATH_LOGCONF "/etc/rsyslog.conf" #endif @@ -224,8 +206,6 @@ static pid_t myPid; /* our pid for use in self-generated messages, e.g. on start /* mypid is read-only after the initial fork() */ static int bHadHUP = 0; /* did we have a HUP? */ -static int bParseHOSTNAMEandTAG = 1; /* global config var: should the hostname and tag be - * parsed inside message - rgerhards, 2006-03-13 */ static int bFinished = 0; /* used by termination signal handler, read-only except there * is either 0 or the number of the signal that requested the * termination. @@ -250,7 +230,6 @@ typedef struct legacyOptsLL_s { legacyOptsLL_t *pLegacyOptsLL = NULL; /* global variables for config file state */ -int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ int iCompatibilityMode = 0; /* version we should be compatible with; 0 means sysklogd. It is the default, so if no -c<n> option is given, we make ourselvs as compatible to sysklogd as possible. */ @@ -259,13 +238,10 @@ static int bLogStatusMsgs = DFLT_bLogStatusMsgs; /* log rsyslog start/stop/HUP m static int bDebugPrintTemplateList = 1;/* output template list in debug mode? */ static int bDebugPrintCfSysLineHandlerList = 1;/* output cfsyslinehandler list in debug mode? */ static int bDebugPrintModuleList = 1;/* output module list in debug mode? */ -uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ -int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ -int bEscapeTab = 1; /* treat tab as escape control character: 0 - no, 1 - yes */ static int bErrMsgToStderr = 1; /* print error messages to stderr (in addition to everything else)? */ int bReduceRepeatMsgs; /* reduce repeated message - 0 - no, 1 - yes */ +int bAbortOnUncleanConfig = 0; /* abort run (rather than starting with partial config) if there was any issue in conf */ int bActExecWhenPrevSusp; /* execute action only when previous one was suspended? */ -int iActExecOnceInterval = 0; /* execute action once every nn seconds */ /* end global config file state variables */ int MarkInterval = 20 * 60; /* interval between marks in seconds - read-only after startup */ @@ -282,7 +258,7 @@ extern int errno; static uchar *pszConfDAGFile = NULL; /* name of config DAG file, non-NULL means generate one */ /* main message queue and its configuration parameters */ -static qqueue_t *pMsgQueue = NULL; /* the main message queue */ +qqueue_t *pMsgQueue = NULL; /* the main message queue */ static int iMainMsgQueueSize = 10000; /* size of the main message queue above */ static int iMainMsgQHighWtrMark = 8000; /* high water mark for disk-assisted queues */ static int iMainMsgQLowWtrMark = 2000; /* low water mark for disk-assisted queues */ @@ -301,54 +277,24 @@ static int iMainMsgQtoWrkShutdown = 60000; /* timeout for worker thread shutdo static int iMainMsgQWrkMinMsgs = 100; /* minimum messages per worker needed to start a new one */ static int iMainMsgQDeqSlowdown = 0; /* dequeue slowdown (simple rate limiting) */ static int64 iMainMsgQueMaxDiskSpace = 0; /* max disk space allocated 0 ==> unlimited */ +static int64 iMainMsgQueDeqBatchSize = 32; /* dequeue batch size */ static int bMainMsgQSaveOnShutdown = 1; /* save queue on shutdown (when DA enabled)? */ static int iMainMsgQueueDeqtWinFromHr = 0; /* hour begin of time frame when queue is to be dequeued */ static int iMainMsgQueueDeqtWinToHr = 25; /* hour begin of time frame when queue is to be dequeued */ -/* support for simple textual representation of FIOP names - * rgerhards, 2005-09-27 - */ -char* -getFIOPName(unsigned iFIOP) -{ - char *pRet; - switch(iFIOP) { - case FIOP_CONTAINS: - pRet = "contains"; - break; - case FIOP_ISEQUAL: - pRet = "isequal"; - break; - case FIOP_STARTSWITH: - pRet = "startswith"; - break; - case FIOP_REGEX: - pRet = "regex"; - break; - default: - pRet = "NOP"; - break; - } - return pRet; -} - - /* Reset config variables to default values. * rgerhards, 2007-07-17 */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { - cCCEscapeChar = '#'; bLogStatusMsgs = DFLT_bLogStatusMsgs; bActExecWhenPrevSusp = 0; - iActExecOnceInterval = 0; bDebugPrintTemplateList = 1; bDebugPrintCfSysLineHandlerList = 1; bDebugPrintModuleList = 1; - bEscapeCCOnRcv = 1; /* default is to escape control characters */ - bEscapeTab = 1; bReduceRepeatMsgs = 0; + bAbortOnUncleanConfig = 0; free(pszMainMsgQFName); pszMainMsgQFName = NULL; iMainMsgQueueSize = 10000; @@ -369,6 +315,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bMainMsgQSaveOnShutdown = 1; MainMsgQueType = QUEUETYPE_FIXED_ARRAY; iMainMsgQueMaxDiskSpace = 0; + iMainMsgQueDeqBatchSize = 32; glbliActionResumeRetryCount = 0; return RS_RET_OK; @@ -378,15 +325,16 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a /* hardcoded standard templates (used for defaults) */ static uchar template_DebugFormat[] = "\"Debug line with all properties:\nFROMHOST: '%FROMHOST%', fromhost-ip: '%fromhost-ip%', HOSTNAME: '%HOSTNAME%', PRI: %PRI%,\nsyslogtag '%syslogtag%', programname: '%programname%', APP-NAME: '%APP-NAME%', PROCID: '%PROCID%', MSGID: '%MSGID%',\nTIMESTAMP: '%TIMESTAMP%', STRUCTURED-DATA: '%STRUCTURED-DATA%',\nmsg: '%msg%'\nescaped msg: '%msg:::drop-cc%'\ninputname: %inputname% rawmsg: '%rawmsg%'\n\n\""; static uchar template_SyslogProtocol23Format[] = "\"<%PRI%>1 %TIMESTAMP:::date-rfc3339% %HOSTNAME% %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA% %msg%\n\""; -static uchar template_TraditionalFileFormat[] = "\"%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n\""; -static uchar template_FileFormat[] = "\"%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n\""; +static uchar template_TraditionalFileFormat[] = "=RSYSLOG_TraditionalFileFormat"; +static uchar template_FileFormat[] = "=RSYSLOG_FileFormat"; +static uchar template_ForwardFormat[] = "=RSYSLOG_ForwardFormat"; +static uchar template_TraditionalForwardFormat[] = "=RSYSLOG_TraditionalForwardFormat"; static uchar template_WallFmt[] = "\"\r\n\7Message from syslogd@%HOSTNAME% at %timegenerated% ...\r\n %syslogtag%%msg%\n\r\""; -static uchar template_ForwardFormat[] = "\"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%\""; -static uchar template_TraditionalForwardFormat[] = "\"<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%\""; static uchar template_StdUsrMsgFmt[] = "\" %syslogtag%%msg%\n\r\""; static uchar template_StdDBFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-mysql%', '%timegenerated:::date-mysql%', %iut%, '%syslogtag%')\",SQL"; static uchar template_StdPgSQLFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%')\",STDSQL"; -/* end template */ +static uchar template_spoofadr[] = "\"%fromhost-ip%\""; +/* end templates */ /* up to the next comment, prototypes that should be removed by reordering */ @@ -395,7 +343,6 @@ static char **crunch_list(char *list); static void reapchild(); static void debug_switch(); static void sighup_handler(); -static void processImInternal(void); static int usage(void) @@ -403,7 +350,7 @@ static int usage(void) fprintf(stderr, "usage: rsyslogd [-c<version>] [-46AdnqQvwx] [-l<hostlist>] [-s<domainlist>]\n" " [-f<conffile>] [-i<pidfile>] [-N<level>] [-M<module load path>]\n" " [-u<number>]\n" - "To run rsyslogd in native mode, use \"rsyslogd -c3 <other options>\"\n\n" + "To run rsyslogd in native mode, use \"rsyslogd -c5 <other options>\"\n\n" "For further information see http://www.rsyslog.com/doc\n"); exit(1); /* "good" exit - done to terminate usage() */ } @@ -415,6 +362,8 @@ static int usage(void) */ /* return back the approximate current number of messages in the main message queue + * This number includes the messages that reside in an associated DA queue (if + * it exists) -- rgerhards, 2009-10-14 */ rsRetVal diagGetMainMsgQSize(int *piSize) @@ -431,7 +380,7 @@ diagGetMainMsgQSize(int *piSize) /* rgerhards, 2005-10-24: crunch_list is called only during option processing. So - * it is never called once rsyslogd is running (not even when HUPed). This code + * it is never called once rsyslogd is running. This code * contains some exits, but they are considered safe because they only happen * during startup. Anyhow, when we review the code here, we might want to * reconsider the exit()s. @@ -459,7 +408,7 @@ static char **crunch_list(char *list) for (count=i=0; p[i]; i++) if (p[i] == LIST_DELIMITER) count++; - if ((result = (char **)malloc(sizeof(char *) * (count+2))) == NULL) { + if ((result = (char **)MALLOC(sizeof(char *) * (count+2))) == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ } @@ -471,7 +420,7 @@ static char **crunch_list(char *list) */ count = 0; while ((q=strchr(p, LIST_DELIMITER))) { - result[count] = (char *) malloc((q - p + 1) * sizeof(char)); + result[count] = (char *) MALLOC((q - p + 1) * sizeof(char)); if (result[count] == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ @@ -482,7 +431,7 @@ static char **crunch_list(char *list) count++; } if ((result[count] = \ - (char *)malloc(sizeof(char) * strlen(p) + 1)) == NULL) { + (char *)MALLOC(sizeof(char) * strlen(p) + 1)) == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ } @@ -528,52 +477,21 @@ void untty(void) #endif -/* Take a raw input line, decode the message, and print the message - * on the appropriate log files. - * rgerhards 2004-11-08: Please note - * that this function does only a partial decoding. At best, it splits - * the PRI part. No further decode happens. The rest is done in - * logmsg(). - * Added the iSource parameter so that we know if we have to parse - * HOSTNAME or not. rgerhards 2004-11-16. - * changed parameter iSource to bParseHost. For details, see comment in - * printchopped(). rgerhards 2005-10-06 - * rgerhards: 2008-03-06: added "flags" to allow an input module to specify - * flags, most importantly to request ignoring the messages' timestamp. - * - * rgerhards, 2008-03-19: - * I added an additional calling parameter to permit specifying the flow - * control capability of the source. - * - * rgerhards, 2008-05-16: - * I added an additional calling parameter (hnameIP) to enable specifying the IP - * of a remote host. - * - * rgerhards, 2008-09-11: - * Interface change: added new parameter "InputName", permits the input to provide - * a string that identifies it. May be NULL, but must be a valid char* pointer if - * non-NULL. - * - * rgerhards, 2008-10-06: - * Interface change: added new parameter "stTime", which enables the caller to provide - * a timestamp that is to be used as timegenerated instead of the current system time. - * This is meant to facilitate performance optimization. Some inputs support such modes. - * If stTime is NULL, the current system time is used. - * - * rgerhards, 2008-10-09: - * interface change: bParseHostname removed, now in flags +/* This takes a received message that must be decoded and submits it to + * the main message queue. This is a legacy function which is being provided + * to aid older input plugins that do not support message creation via + * the new interfaces themselves. It is not recommended to use this + * function for new plugins. -- rgerhards, 2009-10-12 */ -static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int flags, flowControl_t flowCtlType, +rsRetVal +parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType, prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime) { - DEFiRet; - register uchar *p; - int pri; + prop_t *pProp = NULL; msg_t *pMsg; - prop_t *propFromHost = NULL; - prop_t *propFromHostIP = NULL; + DEFiRet; - /* Now it is time to create the message object (rgerhards) */ + /* we now create our own message object and submit it to the queue */ if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { @@ -581,274 +499,17 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f } if(pInputName != NULL) MsgSetInputName(pMsg, pInputName); + MsgSetRawMsg(pMsg, (char*)msg, len); MsgSetFlowControlType(pMsg, flowCtlType); - MsgSetRawMsgWOSize(pMsg, (char*)msg); - - /* test for special codes */ - pri = DEFUPRI; - p = msg; - if (*p == '<') { - pri = 0; - while (isdigit((int) *++p)) - { - pri = 10 * pri + (*p - '0'); - } - if (*p == '>') - ++p; - } - if (pri &~ (LOG_FACMASK|LOG_PRIMASK)) - pri = DEFUPRI; - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); - - /* Now we look at the HOSTNAME. That is a bit complicated... - * If we have a locally received message, it does NOT - * contain any hostname information in the message itself. - * As such, the HOSTNAME is the same as the system that - * the message was received from (that, for obvious reasons, - * being the local host). rgerhards 2004-11-16 - */ - if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) - MsgSetHOSTNAME(pMsg, hname, ustrlen(hname)); - MsgSetRcvFromStr(pMsg, hname, ustrlen(hname), &propFromHost); - CHKiRet(MsgSetRcvFromIPStr(pMsg, hnameIP, ustrlen(hnameIP), &propFromHostIP)); - MsgSetAfterPRIOffs(pMsg, p - msg); - prop.Destruct(&propFromHost); - prop.Destruct(&propFromHostIP); - - logmsg(pMsg, flags); - -finalize_it: - RETiRet; -} + pMsg->msgFlags = flags | NEEDS_PARSING; - -/* This takes a received message that must be decoded and submits it to - * the main message queue. The function calls the necessary parser. - * - * rgerhards, 2006-11-30: I have greatly changed this function. Formerly, - * it tried to reassemble multi-part messages, which is a legacy stock - * sysklogd concept. In essence, that was that messages not ending with - * \0 were glued together. As far as I can see, this is a sysklogd - * specific feature and, from looking at the code, seems to be used - * pretty seldom (if at all). I remove this now, not the least because it is totally - * incompatible with upcoming IETF syslog standards. If you experience - * strange behaviour with messages beeing split across multiple lines, - * this function here might be the place to look at. - * - * Some previous history worth noting: - * I added the "iSource" parameter. This is needed to distinguish between - * messages that have a hostname in them (received from the internet) and - * those that do not have (most prominently /dev/log). rgerhards 2004-11-16 - * And now I removed the "iSource" parameter and changed it to be "bParseHost", - * because all that it actually controls is whether the host is parsed or not. - * For rfc3195 support, we needed to modify the algo for host parsing, so we can - * no longer rely just on the source (rfc3195d forwarded messages arrive via - * unix domain sockets but contain the hostname). rgerhards, 2005-10-06 - * - * rgerhards, 2008-02-18: - * This function was previously called "printchopped"() and has been renamed - * as part of the effort to create a clean internal message submission interface. - * It also has been adopted to our usual calling interface, but currently does - * not provide any useful return states. But we now have the hook and things can - * improve in the future. <-- TODO! - * - * rgerhards, 2008-03-19: - * I added an additional calling parameter to permit specifying the flow - * control capability of the source. - * - * rgerhards, 2008-05-16: - * I added an additional calling parameter (hnameIP) to enable specifying the IP - * of a remote host. - * - * rgerhards, 2008-09-11: - * Interface change: added new parameter "InputName", permits the input to provide - * a string that identifies it. May be NULL, but must be a valid char* pointer if - * non-NULL. - * - * rgerhards, 2008-10-06: - * Interface change: added new parameter "stTime", which enables the caller to provide - * a timestamp that is to be used as timegenerated instead of the current system time. - * This is meant to facilitate performance optimization. Some inputs support such modes. - * If stTime is NULL, the current system time is used. - * - * rgerhards, 2008-10-09: - * interface change: bParseHostname removed, now in flags - */ -rsRetVal -parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType, - prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime) -{ - DEFiRet; - register int iMsg; - uchar *pMsg; - uchar *pData; - uchar *pEnd; - int iMaxLine; - uchar *tmpline = NULL; -# ifdef USE_NETZIP - uchar *deflateBuf = NULL; - uLongf iLenDefBuf; -# endif - - assert(hname != NULL); - assert(hnameIP != NULL); - assert(msg != NULL); - assert(len >= 0); - - /* we first allocate work buffers large enough to hold the configured maximum - * size of a message. Over time, we should change this to a more optimal way, i.e. - * by calling the function with the actual length of the message to be parsed. - * rgerhards, 2008-09-02 - * - * TODO: optimize buffer handling */ - iMaxLine = glbl.GetMaxLine(); - CHKmalloc(tmpline = malloc(sizeof(uchar) * (iMaxLine + 1))); - - /* we first check if we have a NUL character at the very end of the - * message. This seems to be a frequent problem with a number of senders. - * So I have now decided to drop these NULs. However, if they are intentional, - * that may cause us some problems, e.g. with syslog-sign. On the other hand, - * current code always has problems with intentional NULs (as it needs to escape - * them to prevent problems with the C string libraries), so that does not - * really matter. Just to be on the save side, we'll log destruction of such - * NULs in the debug log. - * rgerhards, 2007-09-14 - */ - if(*(msg + len - 1) == '\0') { - DBGPRINTF("dropped NUL at very end of message\n"); - len--; - } - - /* then we check if we need to drop trailing LFs, which often make - * their way into syslog messages unintentionally. In order to remain - * compatible to recent IETF developments, we allow the user to - * turn on/off this handling. rgerhards, 2007-07-23 - */ - if(bDropTrailingLF && *(msg + len - 1) == '\n') { - DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); - len--; - } - - iMsg = 0; /* initialize receiving buffer index */ - pMsg = tmpline; /* set receiving buffer pointer */ - pData = msg; /* set source buffer pointer */ - pEnd = msg + len; /* this is one off, which is intensional */ - -# ifdef USE_NETZIP - /* we first need to check if we have a compressed record. If so, - * we must decompress it. - */ - if(len > 0 && *msg == 'z') { /* compressed data present? (do NOT change order if conditions!) */ - /* we have compressed data, so let's deflate it. We support a maximum - * message size of iMaxLine. If it is larger, an error message is logged - * and the message is dropped. We do NOT try to decompress larger messages - * as such might be used for denial of service. It might happen to later - * builds that such functionality be added as an optional, operator-configurable - * feature. - */ - int ret; - iLenDefBuf = iMaxLine; - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); - ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); - DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", - ret, (long) iLenDefBuf, len-1); - /* Now check if the uncompression worked. If not, there is not much we can do. In - * that case, we log an error message but ignore the message itself. Storing the - * compressed text is dangerous, as it contains control characters. So we do - * not do this. If someone would like to have a copy, this code here could be - * modified to do a hex-dump of the buffer in question. We do not include - * this functionality right now. - * rgerhards, 2006-12-07 - */ - if(ret != Z_OK) { - errmsg.LogError(0, NO_ERRCODE, "Uncompression of a message failed with return code %d " - "- enable debug logging if you need further information. " - "Message ignored.", ret); - FINALIZE; /* unconditional exit, nothing left to do... */ - } - pData = deflateBuf; - pEnd = deflateBuf + iLenDefBuf; - } -# else /* ifdef USE_NETZIP */ - /* in this case, we still need to check if the message is compressed. If so, we must - * tell the user we can not accept it. - */ - if(len > 0 && *msg == 'z') { - errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " - "support enabled. The message will be ignored."); - FINALIZE; - } -# endif /* ifdef USE_NETZIP */ - - while(pData < pEnd) { - if(iMsg >= iMaxLine) { - /* emergency, we now need to flush, no matter if - * we are at end of message or not... - */ - if(iMsg == iMaxLine) { - *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, flags, flowCtlType, pInputName, stTime, ttGenTime); - } else { - /* This case in theory never can happen. If it happens, we have - * a logic error. I am checking for it, because if I would not, - * we would address memory invalidly with the code above. I - * do not care much about this case, just a debug log entry - * (I couldn't do any more smart things anyway...). - * rgerhards, 2007-9-20 - */ - DBGPRINTF("internal error: iMsg > max msg size in parseAndSubmitMessage()\n"); - } - FINALIZE; /* in this case, we are done... nothing left we can do */ - } - if(*pData == '\0') { /* guard against \0 characters... */ - /* changed to the sequence (somewhat) proposed in - * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 - */ - if(iMsg + 3 < iMaxLine) { /* do we have space? */ - *(pMsg + iMsg++) = cCCEscapeChar; - *(pMsg + iMsg++) = '0'; - *(pMsg + iMsg++) = '0'; - *(pMsg + iMsg++) = '0'; - } /* if we do not have space, we simply ignore the '\0'... */ - /* log an error? Very questionable... rgerhards, 2006-11-30 */ - /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ - ++pData; - } else if(bEscapeCCOnRcv && iscntrl((int) *pData) && (*pData != '\t' || bEscapeTab)) { - /* we are configured to escape control characters. Please note - * that this most probably break non-western character sets like - * Japanese, Korean or Chinese. rgerhards, 2007-07-17 - * Note: sysklogd logs octal values only for DEL and CCs above 127. - * For others, it logs ^n where n is the control char converted to an - * alphabet character. We like consistency and thus escape it to octal - * in all cases. If someone complains, we may change the mode. At least - * we known now what's going on. - * rgerhards, 2007-07-17 - */ - if(iMsg + 3 < iMaxLine) { /* do we have space? */ - *(pMsg + iMsg++) = cCCEscapeChar; - *(pMsg + iMsg++) = '0' + ((*pData & 0300) >> 6); - *(pMsg + iMsg++) = '0' + ((*pData & 0070) >> 3); - *(pMsg + iMsg++) = '0' + ((*pData & 0007)); - } /* again, if we do not have space, we ignore the char - see comment at '\0' */ - ++pData; - } else { - *(pMsg + iMsg++) = *pData++; - } - } - - *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - - /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, flags, flowCtlType, pInputName, stTime, ttGenTime); + MsgSetRcvFromStr(pMsg, hname, ustrlen(hname), &pProp); + CHKiRet(prop.Destruct(&pProp)); + CHKiRet(MsgSetRcvFromIPStr(pMsg, hnameIP, ustrlen(hnameIP), &pProp)); + CHKiRet(prop.Destruct(&pProp)); + CHKiRet(submitMsg(pMsg)); finalize_it: - if(tmpline != NULL) - free(tmpline); -# ifdef USE_NETZIP - if(deflateBuf != NULL) - free(deflateBuf); -# endif RETiRet; } @@ -867,13 +528,7 @@ submitErrMsg(int iErr, uchar *msg) /* rgerhards 2004-11-09: the following is a function that can be used - * to log a message orginating from the syslogd itself. In sysklogd code, - * this is done by simply calling logmsg(). However, logmsg() is changed in - * rsyslog so that it takes a msg "object". So it can no longer be called - * directly. This method here solves the need. It provides an interface that - * allows to construct a locally-generated message. Please note that this - * function here probably is only an interim solution and that we need to - * think on the best way to do this. + * to log a message orginating from the syslogd itself. */ rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags) @@ -888,6 +543,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); MsgSetRcvFromIP(pMsg, pLocalHostIP); + MsgSetMSGoffs(pMsg, 0); /* check if we have an error code associated and, if so, * adjust the tag. -- rgerhards, 2008-06-27 */ @@ -900,8 +556,8 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - pMsg->bParseHOSTNAME = 0; flags |= INTERNAL_MSG; + pMsg->msgFlags = flags; /* we now check if we should print internal messages out to stderr. This was * suggested by HKS as a way to help people troubleshoot rsyslog configuration @@ -917,517 +573,196 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) } if(bHaveMainQueue == 0) { /* not yet in queued mode */ - iminternalAddMsg(pri, pMsg, flags); + iminternalAddMsg(pri, pMsg); } else { /* we have the queue, so we can simply provide the * message to the queue engine. */ - logmsg(pMsg, flags); + submitMsg(pMsg); } finalize_it: RETiRet; } - -/* The consumer of dequeued messages. This function is called by the - * queue engine on dequeueing of a message. It runs on a SEPARATE - * THREAD. - * Please note: the message object is destructed by the queue itself! - */ -static rsRetVal -msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) -{ - DEFiRet; - msg_t *pMsg = (msg_t*) pUsr; - - assert(pMsg != NULL); - - if((pMsg->msgFlags & NEEDS_PARSING) != 0) { - parseMsg(pMsg); - } - ruleset.ProcessMsg(pMsg); - msgDestruct(&pMsg); - - RETiRet; -} - - -/* Helper to parseRFCSyslogMsg. This function parses a field up to - * (and including) the SP character after it. The field contents is - * returned in a caller-provided buffer. The parsepointer is advanced - * to after the terminating SP. The caller must ensure that the - * provided buffer is large enough to hold the to be extracted value. - * Returns 0 if everything is fine or 1 if either the field is not - * SP-terminated or any other error occurs. -- rger, 2005-11-24 - * The function now receives the size of the string and makes sure - * that it does not process more than that. The *pLenStr counter is - * updated on exit. -- rgerhards, 2009-09-23 +/* check message against ACL set + * rgerhards, 2009-11-16 */ -static int parseRFCField(uchar **pp2parse, uchar *pResult, int *pLenStr) -{ - uchar *p2parse; - int iRet = 0; - - assert(pp2parse != NULL); - assert(*pp2parse != NULL); - assert(pResult != NULL); - - p2parse = *pp2parse; - - /* this is the actual parsing loop */ - while(*pLenStr > 0 && *p2parse != ' ') { - *pResult++ = *p2parse++; - --(*pLenStr); - } - - if(*pLenStr > 0 && *p2parse == ' ') { - ++p2parse; /* eat SP, but only if not at end of string */ - --(*pLenStr); - } else { - iRet = 1; /* there MUST be an SP! */ +#if 0 +static inline rsRetVal +chkMsgAgainstACL() { + /* if we reach this point, we had a good receive and can process the packet received */ + /* check if we have a different sender than before, if so, we need to query some new values */ + if(net.CmpHost(&frominet, frominetPrev, socklen) != 0) { + CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us + * syslog messages. If it isn't, we do not further + * process the message but log a warning (if we are + * configured to do this). + * rgerhards, 2005-09-26 + */ + *pbIsPermitted = net.isAllowedSender((uchar*)"UDP", + (struct sockaddr *)&frominet, (char*)fromHostFQDN); + + if(!*pbIsPermitted) { + DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); + if(glbl.GetOption_DisallowWarning) { + time_t tt; + + datetime.GetTime(&tt); + if(tt > ttLastDiscard + 60) { + ttLastDiscard = tt; + errmsg.LogError(0, NO_ERRCODE, + "UDP message from disallowed sender %s discarded", + (char*)fromHost); + } + } + } } - *pResult = '\0'; - - /* set the new parse pointer */ - *pp2parse = p2parse; - return 0; } +#endif -/* Helper to parseRFCSyslogMsg. This function parses the structured - * data field of a message. It does NOT parse inside structured data, - * just gets the field as whole. Parsing the single entities is left - * to other functions. The parsepointer is advanced - * to after the terminating SP. The caller must ensure that the - * provided buffer is large enough to hold the to be extracted value. - * Returns 0 if everything is fine or 1 if either the field is not - * SP-terminated or any other error occurs. -- rger, 2005-11-24 - * The function now receives the size of the string and makes sure - * that it does not process more than that. The *pLenStr counter is - * updated on exit. -- rgerhards, 2009-09-23 +/* preprocess a batch of messages, that is ready them for actual processing. This is done + * as a first stage and totally in parallel to any other worker active in the system. So + * it helps us keep up the overall concurrency level. + * rgerhards, 2010-06-09 */ -static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr) -{ - uchar *p2parse; - int bCont = 1; - int iRet = 0; - int lenStr; - - assert(pp2parse != NULL); - assert(*pp2parse != NULL); - assert(pResult != NULL); - - p2parse = *pp2parse; - lenStr = *pLenStr; - - /* this is the actual parsing loop - * Remeber: structured data starts with [ and includes any characters - * until the first ] followed by a SP. There may be spaces inside - * structured data. There may also be \] inside the structured data, which - * do NOT terminate an element. - */ - if(lenStr == 0 || *p2parse != '[') - return 1; /* this is NOT structured data! */ +static inline rsRetVal +preprocessBatch(batch_t *pBatch) { + uchar fromHost[NI_MAXHOST]; + uchar fromHostIP[NI_MAXHOST]; + uchar fromHostFQDN[NI_MAXHOST]; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; + int bSingleRuleset; + ruleset_t *batchRuleset; /* the ruleset used for all message inside the batch, if there is a single one */ + int bIsPermitted; + msg_t *pMsg; + int i; + rsRetVal localRet; + DEFiRet; - if(*p2parse == '-') { /* empty structured data? */ - *pResult++ = '-'; - ++p2parse; - --lenStr; - } else { - while(bCont) { - if(lenStr < 2) { - /* we now need to check if we have only structured data */ - if(lenStr > 0 && *p2parse == ']') { - *pResult++ = *p2parse; - p2parse++; - lenStr--; - bCont = 0; - } else { - iRet = 1; /* this is not valid! */ - bCont = 0; - } - } else if(*p2parse == '\\' && *(p2parse+1) == ']') { - /* this is escaped, need to copy both */ - *pResult++ = *p2parse++; - *pResult++ = *p2parse++; - lenStr -= 2; - } else if(*p2parse == ']' && *(p2parse+1) == ' ') { - /* found end, just need to copy the ] and eat the SP */ - *pResult++ = *p2parse; - p2parse += 2; - lenStr -= 2; - bCont = 0; + bSingleRuleset = 1; + batchRuleset = (pBatch->nElem > 0) ? ((msg_t*) pBatch->pElem[0].pUsrp)->pRuleset : NULL; + + for(i = 0 ; i < pBatch->nElem && !*(pBatch->pbShutdownImmediate) ; i++) { + pMsg = (msg_t*) pBatch->pElem[i].pUsrp; + if((pMsg->msgFlags & NEEDS_ACLCHK_U) != 0) { + DBGPRINTF("msgConsumer: UDP ACL must be checked for message (hostname-based)\n"); + if(net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP) != RS_RET_OK) + continue; + bIsPermitted = net.isAllowedSender2((uchar*)"UDP", + (struct sockaddr *)pMsg->rcvFrom.pfrominet, (char*)fromHostFQDN, 1); + if(!bIsPermitted) { + DBGPRINTF("Message from '%s' discarded, not a permitted sender host\n", + fromHostFQDN); + pBatch->pElem[i].state = BATCH_STATE_DISC; } else { - *pResult++ = *p2parse++; - --lenStr; + /* save some of the info we obtained */ + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); + pMsg->msgFlags &= ~NEEDS_ACLCHK_U; } } - } - - if(lenStr > 0 && *p2parse == ' ') { - ++p2parse; /* eat SP, but only if not at end of string */ - --lenStr; - } else { - iRet = 1; /* there MUST be an SP! */ - } - *pResult = '\0'; - - /* set the new parse pointer */ - *pp2parse = p2parse; - *pLenStr = lenStr; - return 0; -} - -/* parse a RFC5424-formatted syslog message. This function returns - * 0 if processing of the message shall continue and 1 if something - * went wrong and this messe should be ignored. This function has been - * implemented in the effort to support syslog-protocol. Please note that - * the name (parse *RFC*) stems from the hope that syslog-protocol will - * some time become an RFC. Do not confuse this with informational - * RFC 3164 (which is legacy syslog). - * - * currently supported format: - * - * <PRI>VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP [SD-ID]s SP MSG - * - * <PRI> is already stripped when this function is entered. VERSION already - * has been confirmed to be "1", but has NOT been stripped from the message. - * - * rger, 2005-11-24 - */ -int parseRFCSyslogMsg(msg_t *pMsg, int flags) -{ - uchar *p2parse; - uchar *pBuf; - int lenMsg; - int bContParse = 1; - - BEGINfunc - assert(pMsg != NULL); - assert(pMsg->pszRawMsg != NULL); - p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ - lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; - - /* do a sanity check on the version and eat it (the caller checked this already) */ - assert(p2parse[0] == '1' && p2parse[1] == ' '); - p2parse += 2; - lenMsg -= 2; - - /* Now get us some memory we can use as a work buffer while parsing. - * We simply allocated a buffer sufficiently large to hold all of the - * message, so we can not run into any troubles. I think this is - * more wise then to use individual buffers. - */ - if((pBuf = malloc(sizeof(uchar) * (lenMsg + 1))) == NULL) - return 1; - - /* IMPORTANT NOTE: - * Validation is not actually done below nor are any errors handled. I have - * NOT included this for the current proof of concept. However, it is strongly - * advisable to add it when this code actually goes into production. - * rgerhards, 2005-11-24 - */ - - /* TIMESTAMP */ - if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { - if(flags & IGNDATE) { - /* we need to ignore the msg data, so simply copy over reception date */ - memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + if((pMsg->msgFlags & NEEDS_PARSING) != 0) { + if((localRet = parser.ParseMsg(pMsg)) != RS_RET_OK) { + DBGPRINTF("Message discarded, parsing error %d\n", localRet); + pBatch->pElem[i].state = BATCH_STATE_DISC; + } } - } else { - DBGPRINTF("no TIMESTAMP detected!\n"); - bContParse = 0; + if(pMsg->pRuleset != batchRuleset) + bSingleRuleset = 0; } - /* HOSTNAME */ - if(bContParse) { - parseRFCField(&p2parse, pBuf, &lenMsg); - MsgSetHOSTNAME(pMsg, pBuf, ustrlen(pBuf)); - } - - /* APP-NAME */ - if(bContParse) { - parseRFCField(&p2parse, pBuf, &lenMsg); - MsgSetAPPNAME(pMsg, (char*)pBuf); - } + batchSetSingleRuleset(pBatch, bSingleRuleset); - /* PROCID */ - if(bContParse) { - parseRFCField(&p2parse, pBuf, &lenMsg); - MsgSetPROCID(pMsg, (char*)pBuf); - } - - /* MSGID */ - if(bContParse) { - parseRFCField(&p2parse, pBuf, &lenMsg); - MsgSetMSGID(pMsg, (char*)pBuf); - } - - /* STRUCTURED-DATA */ - if(bContParse) { - parseRFCStructuredData(&p2parse, pBuf, &lenMsg); - MsgSetStructuredData(pMsg, (char*)pBuf); - } - - /* MSG */ - MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); - - free(pBuf); - ENDfunc - return 0; /* all ok */ +finalize_it: + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; } - -/* parse a legay-formatted syslog message. This function returns - * 0 if processing of the message shall continue and 1 if something - * went wrong and this messe should be ignored. This function has been - * implemented in the effort to support syslog-protocol. - * rger, 2005-11-24 - * As of 2006-01-10, I am removing the logic to continue parsing only - * when a valid TIMESTAMP is detected. Validity of other fields already - * is ignored. This is due to the fact that the parser has grown smarter - * and is now more able to understand different dialects of the syslog - * message format. I do not expect any bad side effects of this change, - * but I thought I log it in this comment. - * rgerhards, 2006-01-10 +/* The consumer of dequeued messages. This function is called by the + * queue engine on dequeueing of a message. It runs on a SEPARATE + * THREAD. It receives an array of pointers, which it must iterate + * over. We do not do any further batching, as this is of no benefit + * for the main queue. */ -int parseLegacySyslogMsg(msg_t *pMsg, int flags) +static rsRetVal +msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShutdownImmediate) { - uchar *p2parse; - int lenMsg; - int bTAGCharDetected; - int i; /* general index for parsing */ - uchar bufParseTAG[CONF_TAG_MAXSIZE]; - uchar bufParseHOSTNAME[CONF_HOSTNAME_MAXSIZE]; - BEGINfunc - - assert(pMsg != NULL); - assert(pMsg->pszRawMsg != NULL); - lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ - p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ - - /* Check to see if msg contains a timestamp. We start by assuming - * that the message timestamp is the time of reception (which we - * generated ourselfs and then try to actually find one inside the - * message. There we go from high-to low precison and are done - * when we find a matching one. -- rgerhards, 2008-09-16 - */ - if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { - /* we are done - parse pointer is moved by ParseTIMESTAMP3339 */; - } else if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { - /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; - } else if(*p2parse == ' ' && lenMsg > 1) { /* try to see if it is slighly malformed - HP procurve seems to do that sometimes */ - ++p2parse; /* move over space */ - --lenMsg; - if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { - /* indeed, we got it! */ - /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; - } else {/* parse pointer needs to be restored, as we moved it off-by-one - * for this try. - */ - --p2parse; - ++lenMsg; - } - } - - if(flags & IGNDATE) { - /* we need to ignore the msg data, so simply copy over reception date */ - memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); - } - - /* rgerhards, 2006-03-13: next, we parse the hostname and tag. But we - * do this only when the user has not forbidden this. I now introduce some - * code that allows a user to configure rsyslogd to treat the rest of the - * message as MSG part completely. In this case, the hostname will be the - * machine that we received the message from and the tag will be empty. This - * is meant to be an interim solution, but for now it is in the code. - */ - if(bParseHOSTNAMEandTAG && !(flags & INTERNAL_MSG)) { - /* parse HOSTNAME - but only if this is network-received! - * rger, 2005-11-14: we still have a problem with BSD messages. These messages - * do NOT include a host name. In most cases, this leads to the TAG to be treated - * as hostname and the first word of the message as the TAG. Clearly, this is not - * of advantage ;) I think I have now found a way to handle this situation: there - * are certain characters which are frequently used in TAG (e.g. ':'), which are - * *invalid* in host names. So while parsing the hostname, I check for these characters. - * If I find them, I set a simple flag but continue. After parsing, I check the flag. - * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change - * the fields. I think this logic shall work with any type of syslog message. - * rgerhards, 2009-06-23: and I now have extended this logic to every character - * that is not a valid hostname. - */ - bTAGCharDetected = 0; - if(lenMsg > 0 && flags & PARSE_HOSTNAME) { - i = 0; - while(i < lenMsg && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.' - || p2parse[i] == '_' || p2parse[i] == '-') && i < (CONF_HOSTNAME_MAXSIZE - 1)) { - bufParseHOSTNAME[i] = p2parse[i]; - ++i; - } - - if(i == lenMsg) { - /* we have a message that is empty immediately after the hostname, - * but the hostname thus is valid! -- rgerhards, 2010-02-22 - */ - p2parse += i; - lenMsg -= i; - bufParseHOSTNAME[i] = '\0'; - MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); - } else if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) { - /* we got a hostname! */ - p2parse += i + 1; /* "eat" it (including SP delimiter) */ - lenMsg -= i + 1; - bufParseHOSTNAME[i] = '\0'; - MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); - } - } - - /* now parse TAG - that should be present in message from all sources. - * This code is somewhat not compliant with RFC 3164. As of 3164, - * the TAG field is ended by any non-alphanumeric character. In - * practice, however, the TAG often contains dashes and other things, - * which would end the TAG. So it is not desirable. As such, we only - * accept colon and SP to be terminators. Even there is a slight difference: - * a colon is PART of the TAG, while a SP is NOT part of the tag - * (it is CONTENT). Starting 2008-04-04, we have removed the 32 character - * size limit (from RFC3164) on the tag. This had bad effects on existing - * envrionments, as sysklogd didn't obey it either (probably another bug - * in RFC3164...). We now receive the full size, but will modify the - * outputs so that only 32 characters max are used by default. - */ - i = 0; - while(lenMsg > 0 && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) { - bufParseTAG[i++] = *p2parse++; - --lenMsg; - } - if(lenMsg > 0 && *p2parse == ':') { - ++p2parse; - --lenMsg; - bufParseTAG[i++] = ':'; - } - - /* no TAG can only be detected if the message immediatly ends, in which case an empty TAG - * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23 - */ - bufParseTAG[i] = '\0'; /* terminate string */ - MsgSetTAG(pMsg, bufParseTAG, i); - } else {/* we enter this code area when the user has instructed rsyslog NOT - * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 - */ - if(!(flags & INTERNAL_MSG)) { - DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); - } + DEFiRet; + assert(pBatch != NULL); + pBatch->pbShutdownImmediate = pbShutdownImmediate; /* TODO: move this to batch creation! */ + preprocessBatch(pBatch); +//pBatch->bSingleRuleset = 0; // TODO: testing aid, remove!!!! + ruleset.ProcessBatch(pBatch); +//TODO: the BATCH_STATE_COMM must be set somewhere down the road, but we +//do not have this yet and so we emulate -- 2010-06-10 +int i; + for(i = 0 ; i < pBatch->nElem && !*pbShutdownImmediate ; i++) { + pBatch->pElem[i].state = BATCH_STATE_COMM; } - - /* The rest is the actual MSG */ - MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); - - ENDfunc - return 0; /* all ok */ + RETiRet; } /* submit a message to the main message queue. This is primarily * a hook to prevent the need for callers to know about the main message queue - * (which may change in the future as we will probably have multiple rule - * sets and thus queues...). * rgerhards, 2008-02-13 */ rsRetVal submitMsg(msg_t *pMsg) { + qqueue_t *pQueue; + ruleset_t *pRuleset; DEFiRet; ISOBJ_TYPE_assert(pMsg, msg); + pRuleset = MsgGetRuleset(pMsg); + + pQueue = (pRuleset == NULL) ? pMsgQueue : ruleset.GetRulesetQueue(pRuleset); MsgPrepareEnqueue(pMsg); - qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); + qqueueEnqObj(pQueue, pMsg->flowCtlType, (void*) pMsg); RETiRet; } /* submit multiple messages at once, very similar to submitMsg, just - * for multi_submit_t. + * for multi_submit_t. All messages need to go into the SAME queue! * rgerhards, 2009-06-16 */ rsRetVal multiSubmitMsg(multi_submit_t *pMultiSub) { int i; + qqueue_t *pQueue; + ruleset_t *pRuleset; DEFiRet; assert(pMultiSub != NULL); + if(pMultiSub->nElem == 0) + FINALIZE; + for(i = 0 ; i < pMultiSub->nElem ; ++i) { MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); } - iRet = qqueueMultiEnqObj(pMsgQueue, pMultiSub); + pRuleset = MsgGetRuleset(pMultiSub->ppMsgs[0]); + pQueue = (pRuleset == NULL) ? pMsgQueue : ruleset.GetRulesetQueue(pRuleset); + iRet = pQueue->MultiEnq(pQueue, pMultiSub); pMultiSub->nElem = 0; +finalize_it: RETiRet; } -/* Log a message to the appropriate log files, users, etc. based on - * the priority. - * rgerhards 2004-11-08: actually, this also decodes all but the PRI part. - * rgerhards 2004-11-09: ... but only, if syslogd could properly be initialized - * if not, we use emergency logging to the console and in - * this case, no further decoding happens. - * changed to no longer receive a plain message but a msg object instead. - * rgerhards-2004-11-16: OK, we are now up to another change... This method - * actually needs to PARSE the message. How exactly this needs to happen depends on - * a number of things. Most importantly, it depends on the source. For example, - * locally received messages (SOURCE_UNIXAF) do NOT have a hostname in them. So - * we need to treat them differntly form network-received messages which have. - * Well, actually not all network-received message really have a hostname. We - * can just hope they do, but we can not be sure. So this method tries to find - * whatever can be found in the message and uses that... Obviously, there is some - * potential for misinterpretation, which we simply can not solve under the - * circumstances given. - */ -void -logmsg(msg_t *pMsg, int flags) -{ - char *msg; - - BEGINfunc - assert(pMsg != NULL); - assert(pMsg->pszRawMsg != NULL); - - msg = (char*) pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ - DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); - - /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have - * a traditional syslog message or one formatted according to syslog-protocol. - * We need to apply different parsers depending on that. We use the - * -protocol VERSION field for the detection. - */ - if(msg[0] == '1' && msg[1] == ' ') { - DBGPRINTF("Message has syslog-protocol format.\n"); - setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, flags) == 1) { - msgDestruct(&pMsg); - return; - } - } else { /* we have legacy syslog */ - DBGPRINTF("Message has legacy syslog format.\n"); - setProtocolVersion(pMsg, 0); - if(parseLegacySyslogMsg(pMsg, flags) == 1) { - msgDestruct(&pMsg); - return; - } - } - - /* ---------------------- END PARSING ---------------- */ - - /* now submit the message to the main queue - then we are done */ - pMsg->msgFlags = flags; - MsgPrepareEnqueue(pMsg); - qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); - ENDfunc -} static void @@ -1460,7 +795,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) * the beginn of the llExec(). This makes it slightly less correct, but * in an acceptable way. -- rgerhards, 2008-09-16 */ - if (pAction->f_prevcount && time(NULL) >= REPEATTIME(pAction)) { + if (pAction->f_prevcount && datetime.GetTime(NULL) >= REPEATTIME(pAction)) { DBGPRINTF("flush %s: repeated %d times, %d sec.\n", module.GetStateName(pAction->pMod), pAction->f_prevcount, repeatinterval[pAction->f_repeatcount]); @@ -1489,7 +824,7 @@ static void debug_switch() struct tm tp; struct sigaction sigAct; - time(&tTime); + datetime.GetTime(&tTime); localtime_r(&tTime, &tp); if(debugging_on == 0) { debugging_on = 1; @@ -1520,7 +855,7 @@ void legacyOptsEnq(uchar *line) { legacyOptsLL_t *pNew; - pNew = malloc(sizeof(legacyOptsLL_t)); + pNew = MALLOC(sizeof(legacyOptsLL_t)); if(line == NULL) pNew->line = NULL; else @@ -1657,6 +992,7 @@ static void doDie(int sig) # define MSG1 "DoDie called.\n" # define MSG2 "DoDie called 5 times - unconditional exit\n" static int iRetries = 0; /* debug aid */ + dbgprintf(MSG1); if(Debug == DEBUG_FULL) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { @@ -1723,6 +1059,7 @@ die(int sig) /* close the inputs */ DBGPRINTF("Terminating input threads...\n"); + glbl.SetGlobalInputTermination(); thrdTerminateAll(); /* and THEN send the termination log message (see long comment above) */ @@ -1755,8 +1092,6 @@ die(int sig) */ tplDeleteAll(); - remove_pid(PidFile); - /* de-init some modules */ modExitIminternal(); @@ -1778,16 +1113,6 @@ die(int sig) /* terminate the remaining classes */ GlobalClassExit(); - /* TODO: this would also be the right place to de-init the builtin output modules. We - * do not currently do that, because the module interface does not allow for - * it. This will come some time later (it's essential with loadable modules). - * For the time being, this is a memory leak on exit, but as the process is - * terminated, we do not really bother about it. - * rgerhards, 2007-08-03 - * I have added some code now, but all that mod init/de-init should be moved to - * init, so that modules are unloaded and reloaded on HUP to. Eventually it should go - * into destructAllActions() - but that needs to be seen. -- rgerhards, 2007-08-09 - */ module.UnloadAndDestructAll(eMOD_LINK_ALL); DBGPRINTF("Clean shutdown completed, bye\n"); @@ -1800,6 +1125,9 @@ die(int sig) */ freeAllDynMemForTermination(); /* NO CODE HERE - feeelAllDynMemForTermination() must be the last thing before exit()! */ + + remove_pid(PidFile); + exit(0); /* "good" exit, this is the terminator function for rsyslog [die()] */ } @@ -2119,13 +1447,6 @@ static void dbgPrintInitInfo(void) DBGPRINTF("Messages with malicious PTR DNS Records are %sdropped.\n", glbl.GetDropMalPTRMsgs() ? "" : "not "); - DBGPRINTF("Control characters are %sreplaced upon reception.\n", - bEscapeCCOnRcv? "" : "not "); - - if(bEscapeCCOnRcv) - DBGPRINTF("Control character escape sequence prefix is '%c'.\n", - cCCEscapeChar); - DBGPRINTF("Main queue size %d messages.\n", iMainMsgQueueSize); DBGPRINTF("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", iMainMsgQueueNumWorkers, iMainMsgQtoWrkShutdown, iMainMsgQPersistUpdCnt); @@ -2155,6 +1476,7 @@ static rsRetVal runInputModules(void) { modInfo_t *pMod; + int bNeedsCancel; BEGINfunc /* loop through all modules and activate them (brr...) */ @@ -2162,7 +1484,9 @@ runInputModules(void) while(pMod != NULL) { if(pMod->mod.im.bCanRun) { /* activate here */ - thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun); + bNeedsCancel = (pMod->isCompatibleWithFeature(sFEATURENonCancelInputTermination) == RS_RET_OK) ? + 0 : 1; + thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun, bNeedsCancel); } pMod = module.GetNxtType(pMod, eMOD_IN); } @@ -2200,13 +1524,76 @@ startInputModules(void) } -/* INIT -- Initialize syslogd from configuration table - * init() is called at initial startup AND each time syslogd is HUPed +/* create a main message queue, now also used for ruleset queues. This function + * needs to be moved to some other module, but it is considered acceptable for + * the time being (remember that we want to restructure config processing at large!). + * rgerhards, 2009-10-27 + */ +rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName) +{ + DEFiRet; + + /* switch the message object to threaded operation, if necessary */ + if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { + MsgEnableThreadSafety(); + } + + /* create message queue */ + CHKiRet_Hdlr(qqueueConstruct(ppQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { + /* no queue is fatal, we need to give up in that case... */ + errmsg.LogError(0, iRet, "could not create (ruleset) main message queue"); \ + } + /* name our main queue object (it's not fatal if it fails...) */ + obj.SetName((obj_t*) (*ppQueue), pszQueueName); + + /* ... set some properties ... */ +# define setQPROP(func, directive, data) \ + CHKiRet_Hdlr(func(*ppQueue, data)) { \ + errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ + } +# define setQPROPstr(func, directive, data) \ + CHKiRet_Hdlr(func(*ppQueue, data, (data == NULL)? 0 : strlen((char*) data))) { \ + errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ + } + + setQPROP(qqueueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); + setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); + setQPROP(qqueueSetiDeqBatchSize, "$MainMsgQueueDequeueBatchSize", iMainMsgQueDeqBatchSize); + setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); + setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); + setQPROP(qqueueSetbSyncQueueFiles, "$MainMsgQueueSyncQueueFiles", bMainMsgQSyncQeueFiles); + setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); + setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); + setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); + setQPROP(qqueueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); + setQPROP(qqueueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); + setQPROP(qqueueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); + setQPROP(qqueueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); + setQPROP(qqueueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); + setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); + setQPROP(qqueueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); + setQPROP(qqueueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); + setQPROP(qqueueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); + +# undef setQPROP +# undef setQPROPstr + + /* ... and finally start the queue! */ + CHKiRet_Hdlr(qqueueStart(*ppQueue)) { + /* no queue is fatal, we need to give up in that case... */ + errmsg.LogError(0, iRet, "could not start (ruleset) main message queue"); \ + } + RETiRet; +} + + +/* INIT -- Initialize syslogd * Note that if iConfigVerify is set, only the config file is verified but nothing * else happens. -- rgerhards, 2008-07-28 */ static rsRetVal -init() +init(void) { rsRetVal localRet; int iNbrActions; @@ -2217,46 +1604,8 @@ init() struct sigaction sigAct; DEFiRet; - thrdTerminateAll(); /* stop all running input threads - TODO: reconsider location! */ - - /* initialize some static variables */ - pDfltHostnameCmp = NULL; - pDfltProgNameCmp = NULL; - eDfltHostnameCmpMode = HN_NO_COMP; - DBGPRINTF("rsyslog %s - called init()\n", VERSION); - /* delete the message queue, which also flushes all messages left over */ - if(pMsgQueue != NULL) { - DBGPRINTF("deleting main message queue\n"); - qqueueDestruct(&pMsgQueue); /* delete pThis here! */ - pMsgQueue = NULL; - } - - /* Close all open log files and free log descriptor array. This also frees - * all output-modules instance data. - */ - destructAllActions(); - - /* Unload all non-static modules */ - DBGPRINTF("Unloading non-static modules.\n"); - module.UnloadAndDestructAll(eMOD_LINK_DYNAMIC_LOADED); - - DBGPRINTF("Clearing templates.\n"); - tplDeleteNew(); - - /* re-setting values to defaults (where applicable) */ - /* once we have loadable modules, we must re-visit this code. The reason is - * that config variables are not re-set, because the module is not yet loaded. On - * the other hand, that doesn't matter, because the module got unloaded and is then - * re-loaded, so the variables should be re-set via that way. And this is exactly how - * it works. Loadable module's variables are initialized on load, the rest here. - * rgerhards, 2008-04-28 - */ - conf.cfsysline((uchar*)"ResetConfigVariables"); - - conf.ReInitConf(); - /* construct the default ruleset */ ruleset.Construct(&pRuleset); ruleset.SetName(pRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); @@ -2304,17 +1653,6 @@ init() legacyOptsHook(); - /* we are now done with reading the configuration. This is the right time to - * free some objects that were just needed for loading it. rgerhards 2005-10-19 - */ - if(pDfltHostnameCmp != NULL) { - rsCStrDestruct(&pDfltHostnameCmp); - } - - if(pDfltProgNameCmp != NULL) { - rsCStrDestruct(&pDfltProgNameCmp); - } - /* some checks */ if(iMainMsgQueueNumWorkers < 1) { errmsg.LogError(0, NO_ERRCODE, "$MainMsgQueueNumWorkers must be at least 1! Set to 1.\n"); @@ -2350,58 +1688,20 @@ init() ABORT_FINALIZE(RS_RET_VALIDATION_RUN); } - /* switch the message object to threaded operation, if necessary */ - if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { - MsgEnableThreadSafety(); + if(bAbortOnUncleanConfig && bHadConfigErr) { + fprintf(stderr, "rsyslogd: $AbortOnUncleanConfig is set, and config is not clean.\n" + "Check error log for details, fix errors and restart. As a last\n" + "resort, you may want to remove $AbortOnUncleanConfig to permit a\n" + "startup with a dirty config.\n"); + exit(2); } /* create message queue */ - CHKiRet_Hdlr(qqueueConstruct(&pMsgQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { + CHKiRet_Hdlr(createMainQueue(&pMsgQueue, UCHAR_CONSTANT("main Q"))) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); exit(1); } - /* name our main queue object (it's not fatal if it fails...) */ - obj.SetName((obj_t*) pMsgQueue, (uchar*) "main Q"); - - /* ... set some properties ... */ -# define setQPROP(func, directive, data) \ - CHKiRet_Hdlr(func(pMsgQueue, data)) { \ - errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ - } -# define setQPROPstr(func, directive, data) \ - CHKiRet_Hdlr(func(pMsgQueue, data, (data == NULL)? 0 : strlen((char*) data))) { \ - errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ - } - - setQPROP(qqueueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); - setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); - setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); - setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); - setQPROP(qqueueSetbSyncQueueFiles, "$MainMsgQueueSyncQueueFiles", bMainMsgQSyncQeueFiles); - setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); - setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); - setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); - setQPROP(qqueueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); - setQPROP(qqueueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); - setQPROP(qqueueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); - setQPROP(qqueueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); - setQPROP(qqueueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); - setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); - setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); - setQPROP(qqueueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); - setQPROP(qqueueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); - setQPROP(qqueueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); - -# undef setQPROP -# undef setQPROPstr - - /* ... and finally start the queue! */ - CHKiRet_Hdlr(qqueueStart(pMsgQueue)) { - /* no queue is fatal, we need to give up in that case... */ - fprintf(stderr, "fatal error %d: could not start message queue - rsyslogd can not run!\n", iRet); - exit(1); - } bHaveMainQueue = (MainMsgQueType == QUEUETYPE_DIRECT) ? 0 : 1; DBGPRINTF("Main processing queue is initialized and running\n"); @@ -2420,24 +1720,24 @@ init() dbgPrintInitInfo(); } + memset(&sigAct, 0, sizeof (sigAct)); + sigemptyset(&sigAct.sa_mask); + sigAct.sa_handler = sighup_handler; + sigaction(SIGHUP, &sigAct, NULL); + + DBGPRINTF(" started.\n"); + /* we now generate the startup message. It now includes everything to * identify this instance. -- rgerhards, 2005-08-17 */ if(bLogStatusMsgs) { snprintf(bufStartUpMsg, sizeof(bufStartUpMsg)/sizeof(char), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ - "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] (re)start", + "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] start", (int) myPid); logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)bufStartUpMsg, 0); } - memset(&sigAct, 0, sizeof (sigAct)); - sigemptyset(&sigAct.sa_mask); - sigAct.sa_handler = sighup_handler; - sigaction(SIGHUP, &sigAct, NULL); - - DBGPRINTF(" (re)started.\n"); - finalize_it: RETiRet; } @@ -2555,6 +1855,9 @@ void sighup_handler() sigaction(SIGHUP, &sigAct, NULL); } +void sigttin_handler() +{ +} /* this function pulls all internal messages from the buffer * and puts them into the processing engine. @@ -2562,14 +1865,13 @@ void sighup_handler() * really help us. TODO: add error messages? * rgerhards, 2007-08-03 */ -static void processImInternal(void) +static inline void processImInternal(void) { int iPri; - int iFlags; msg_t *pMsg; - while(iminternalRemoveMsg(&iPri, &pMsg, &iFlags) == RS_RET_OK) { - logmsg(pMsg, iFlags); + while(iminternalRemoveMsg(&iPri, &pMsg) == RS_RET_OK) { + submitMsg(pMsg); } } @@ -2600,20 +1902,13 @@ doHUP(void) if(bLogStatusMsgs) { snprintf(buf, sizeof(buf) / sizeof(char), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION - "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed, type '%s'.", - (int) myPid, glbl.GetHUPisRestart() ? "restart" : "lightweight"); + "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed", + (int) myPid); errno = 0; logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)buf, 0); } - if(glbl.GetHUPisRestart()) { - DBGPRINTF("Received SIGHUP, configured to be restart, reloading rsyslogd.\n"); - init(); /* main queue is stopped as part of init() */ - runInputModules(); - } else { - DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); - ruleset.IterateAllActions(doHUPActions, NULL); - } + ruleset.IterateAllActions(doHUPActions, NULL); } @@ -2721,6 +2016,20 @@ static rsRetVal loadBuildInModules(void) */ CHKiRet(module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)); + /* load build-in parser modules */ + CHKiRet(module.doModInit(modInitpmrfc5424, UCHAR_CONSTANT("builtin-pmrfc5424"), NULL)); + CHKiRet(module.doModInit(modInitpmrfc3164, UCHAR_CONSTANT("builtin-pmrfc3164"), NULL)); + + /* and set default parser modules (order is *very* important, legacy (3164) parse needs to go last! */ + CHKiRet(parser.AddDfltParser(UCHAR_CONSTANT("rsyslog.rfc5424"))); + CHKiRet(parser.AddDfltParser(UCHAR_CONSTANT("rsyslog.rfc3164"))); + + /* load build-in strgen modules */ + CHKiRet(module.doModInit(modInitsmfile, UCHAR_CONSTANT("builtin-smfile"), NULL)); + CHKiRet(module.doModInit(modInitsmtradfile, UCHAR_CONSTANT("builtin-smtradfile"), NULL)); + CHKiRet(module.doModInit(modInitsmfwd, UCHAR_CONSTANT("builtin-smfwd"), NULL)); + CHKiRet(module.doModInit(modInitsmtradfwd, UCHAR_CONSTANT("builtin-smtradfwd"), NULL)); + /* ok, initialization of the command handler probably does not 100% belong right in * this space here. However, with the current design, this is actually quite a good * place to put it. We might decide to shuffle it around later, but for the time @@ -2750,18 +2059,15 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeueslowdown", 0, eCmdHdlrInt, NULL, &iMainMsgQDeqSlowdown, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreadminimummessages", 0, eCmdHdlrInt, NULL, &iMainMsgQWrkMinMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuemaxfilesize", 0, eCmdHdlrSize, NULL, &iMainMsgQueMaxFileSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuebatchsize", 0, eCmdHdlrSize, NULL, &iMainMsgQueDeqBatchSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iMainMsgQueMaxDiskSpace, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesaveonshutdown", 0, eCmdHdlrBinary, NULL, &bMainMsgQSaveOnShutdown, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuetimebegin", 0, eCmdHdlrInt, NULL, &iMainMsgQueueDeqtWinFromHr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iMainMsgQueueDeqtWinToHr, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"abortonuncleanconfig", 0, eCmdHdlrBinary, NULL, &bAbortOnUncleanConfig, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgreduction", 0, eCmdHdlrBinary, NULL, &bReduceRepeatMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlywhenpreviousissuspended", 0, eCmdHdlrBinary, NULL, &bActExecWhenPrevSusp, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyonceeveryinterval", 0, eCmdHdlrInt, NULL, &iActExecOnceInterval, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeinterval", 0, eCmdHdlrInt, setActionResumeInterval, NULL, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"template", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_TEMPLATE, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"outchannel", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_OUTCHANNEL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"allowedsender", 0, eCmdHdlrCustomHandler, conf.doNameLine, (void*)DIR_ALLOWEDSENDER, NULL)); @@ -2783,11 +2089,6 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroupid", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); - /* now add other modules handlers (we should work on that to be able to do it in ClassInit(), but so far - * that is not possible). -- rgerhards, 2008-01-28 - */ - CHKiRet(actionAddCfSysLineHdrl()); - finalize_it: RETiRet; } @@ -2847,10 +2148,6 @@ static rsRetVal mainThread() DEFiRet; uchar *pTmp; - /* Note: signals MUST be processed by the thread this code is running in. The reason - * is that we need to interrupt the select() system call. -- rgerhards, 2007-10-17 - */ - /* initialize the build-in templates */ pTmp = template_DebugFormat; tplAddLine("RSYSLOG_DebugFormat", &pTmp); @@ -2872,6 +2169,8 @@ static rsRetVal mainThread() tplAddLine(" StdDBFmt", &pTmp); pTmp = template_StdPgSQLFmt; tplLastStaticInit(tplAddLine(" StdPgSQLFmt", &pTmp)); + pTmp = template_spoofadr; + tplLastStaticInit(tplAddLine("RSYSLOG_omudpspoofDfltSourceTpl", &pTmp)); CHKiRet(init()); @@ -2894,20 +2193,16 @@ static rsRetVal mainThread() */ if(gidDropPriv != 0) { doDropPrivGid(gidDropPriv); - glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ } if(uidDropPriv != 0) { doDropPrivUid(uidDropPriv); - glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ } /* finally let the inputs run... */ runInputModules(); /* END OF INTIALIZATION - * ... but keep in mind that we might do a restart and thus init() might - * be called again. -- rgerhards, 2005-10-24 */ DBGPRINTF("initialization completed, transitioning to regular run mode\n"); @@ -2965,14 +2260,14 @@ InitGlobalClasses(void) CHKiRet(objUse(conf, CORE_COMPONENT)); pErrObj = "prop"; CHKiRet(objUse(prop, CORE_COMPONENT)); + pErrObj = "parser"; + CHKiRet(objUse(parser, CORE_COMPONENT)); /* intialize some dummy classes that are not part of the runtime */ pErrObj = "action"; CHKiRet(actionClassInit()); pErrObj = "template"; CHKiRet(templateInit()); - pErrObj = "parser"; - CHKiRet(parserClassInit()); /* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ pErrObj = "net"; @@ -3011,6 +2306,7 @@ GlobalClassExit(void) objRelease(rule, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ + parserClassExit(); /* this is hack, currently core_modules do not get this automatically called */ objRelease(datetime, CORE_COMPONENT); /* TODO: implement the rest of the deinit */ @@ -3051,7 +2347,7 @@ bufOptAdd(char opt, char *arg) DEFiRet; bufOpt_t *pBuf; - if((pBuf = malloc(sizeof(bufOpt_t))) == NULL) + if((pBuf = MALLOC(sizeof(bufOpt_t))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pBuf->optchar = opt; @@ -3179,6 +2475,8 @@ doGlblProcessInit(void) sigaction(SIGCHLD, &sigAct, NULL); sigAct.sa_handler = Debug ? debug_switch : SIG_IGN; sigaction(SIGUSR1, &sigAct, NULL); + sigAct.sa_handler = sigttin_handler; + sigaction(SIGTTIN, &sigAct, NULL); /* (ab)used to interrupt input threads */ sigAct.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sigAct, NULL); sigaction(SIGXFSZ, &sigAct, NULL); /* do not abort if 2gig file limit is hit */ @@ -3509,7 +2807,7 @@ int realMain(int argc, char **argv) case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); if(iHelperUOpt & 0x01) - bParseHOSTNAMEandTAG = 0; + glbl.SetParseHOSTNAMEandTAG(0); if(iHelperUOpt & 0x02) bChDirRoot = 0; break; diff --git a/tools/syslogd.h b/tools/syslogd.h index 3dfdbe2b..c3b99f9d 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -32,7 +32,6 @@ /* the following prototypes should go away once we have an input * module interface -- rgerhards, 2007-12-12 */ -void logmsg(msg_t *pMsg, int flags); extern int NoHops; extern int send_to_all; extern int Debug; |
