diff options
56 files changed, 1594 insertions, 2391 deletions
@@ -1,4 +1,123 @@ --------------------------------------------------------------------------- +Version 5.9.6 [V5-DEVEL], 2012-??-?? +- refactored imklog linux driver, now combined with BSD driver + The Linux driver no longer supports outdated kernel symbol resolution, + which was disabled by default for very long. Also overall cleanup, + resulting in much smaller code. Linux and BSD are now covered by a + single small driver. +- $IMUXSockRateLimitInterval DEFAULT CHANGED, was 5, now 0 + The new default turns off rate limiting. This was chosen as people + experienced problems with rate-limiting activated by default. Now it + needs an explicit opt-in by setting this parameter. + Thanks to Chris Gaffney for suggesting to make it opt-in; thanks to + many unnamed others who already had complained at the time Chris made + the suggestion ;-) +--------------------------------------------------------------------------- +Version 5.9.5 [V5-DEVEL], 2011-11-29 +- new stats counters for imudp and imtcp +- new stats counters "discarded.nf" and "discarded.full" for queue object. + Tells how many messages have been discarded due to queue full condition. +- enhanced module loader to not rely on PATH_MAX +--------------------------------------------------------------------------- +Version 5.9.4 [V5-DEVEL], 2011-11-29 +- imuxsock: added capability to "annotate" messages with "trusted + information", which contains some properties obtained from the system + and as such sure to not be faked. This is inspired by the similiar idea + introduced in systemd. +- removed dependency on gcrypt for recently-enough GnuTLS + see: http://bugzilla.adiscon.com/show_bug.cgi?id=289 +- bugfix: imuxsock did no longer ignore message-provided timestamp, if + so configured (the *default*). Lead to no longer sub-second timestamps. + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=281 +- bugfix: omfile returns fatal error code for things that go really wrong + previously, RS_RET_RESUME was returned, which lead to a loop inside the + rule engine as omfile could not really recover. +- bugfix: rsyslogd -v always said 64 atomics were not present + thanks to mono_matsuko for the patch +--------------------------------------------------------------------------- +Version 5.9.3 [V5-DEVEL], 2011-09-01 +- bugfix/security: off-by-two bug in legacy syslog parser, CVE-2011-3200 +- bugfix: mark message processing did not work correctly +- added capability to emit config error location info for warnings + otherwise, omusrmsg's warning about new config format was not + accompanied by problem location. +- bugfix: potential misadressing in property replacer +- bugfix: MSGID corruption in RFC5424 parser under some circumstances + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=275 +- bugfix: The NUL-Byte for the syslogtag was not copied in MsgDup (msg.c) +--------------------------------------------------------------------------- +Version 5.9.2 [V5-DEVEL] (rgerhards), 2011-07-11 +- systemd support: set stdout/stderr to null - thx to Lennart for the patch +- added support for the ":omusrmsg:" syntax in configuring user messages +- added support for the ":omfile:" syntax in configuring user messages +--------------------------------------------------------------------------- +Version 5.9.1 [V5-DEVEL] (rgerhards), 2011-06-30 +- added support for obtaining timestamp for kernel message from message + If the kernel time-stamps messages, time is now take from that + timestamp instead of the system time when the message was read. This + provides much better accuracy. Thanks to Lennart Poettering for + suggesting this feature and his help during implementation. +- added support for obtaining timestamp from system for imuxsock + This permits to read the time a message was submitted to the system + log socket. Most importantly, this is provided in microsecond resolution. + So we are able to obtain high precision timestampis even for messages + that were - as is usual - not formatted with them. This also simplifies + things in regard to local time calculation in chroot environments. + Many thanks to Lennart Poettering for suggesting this feature, + providing some guidance on implementing it and coordinating getting the + necessary support into the Linux kernel. +- bugfix: timestamp was incorrectly calculated for timezones with minute + offset + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=271 +- bugfix: problems in failover action handling + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=270 + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=254 +- bugfix: mutex was invalidly left unlocked during action processing + At least one case where this can occur is during thread shutdown, which + may be initiated by lower activity. In most cases, this is quite + unlikely to happen. However, if it does, data structures may be + corrupted which could lead to fatal failure and segfault. I detected + this via a testbench test, not a user report. But I assume that some + users may have had unreproducable aborts that were cause by this bug. +- bugfix: memory leak in imtcp & subsystems under some circumstances + This leak is tied to error conditions which lead to incorrect cleanup + of some data structures. [backport from v6] +- bugfix/improvement:$WorkDirectory now gracefully handles trailing slashes +--------------------------------------------------------------------------- +Version 5.9.0 [V5-DEVEL] (rgerhards), 2011-06-08 +- imfile: added $InputFileMaxLinesAtOnce directive +- enhanced imfile to support input batching +- added capability for imtcp and imptcp to activate keep-alive packets + at the socket layer. This has not been added to imttcp, as the latter is + only an experimental module, and one which did not prove to be useful. + reference: http://kb.monitorware.com/post20791.html +- added support to control KEEPALIVE settings in imptcp + this has not yet been added to imtcp, but could be done on request. +- $ActionName is now also used for naming of queues in impstats + as well as in the debug output +- bugfix: do not open files with full privileges, if privs will be dropped + This make the privilege drop code more bulletproof, but breaks Ubuntu's + work-around for log files created by external programs with the wrong + user and/or group. Note that it was long said that this "functionality" + would break once we go for serious privilege drop code, so hopefully + nobody still depends on it (and, if so, they lost...). +- bugfix: pipes not opened in full priv mode when privs are to be dropped +- this begins a new devel branch for v5 +- better handling of queue i/o errors in disk queues. This is kind of a + bugfix, but a very intrusive one, this it goes into the devel version + first. Right now, "file not found" is handled and leads to the new + emergency mode, in which disk action is stopped and the queue run + in direct mode. An error message is emited if this happens. +- added support for user-level PRI provided via systemd +- added new config directive $InputTCPFlowControl to select if tcp + received messages shall be flagged as light delayable or not. +- enhanced omhdfs to support batching mode. This permits to increase + performance, as we now call the HDFS API with much larger message + sizes and far more infrequently +- bugfix: failover did not work correctly if repeated msg reduction was on + affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 +--------------------------------------------------------------------------- Version 5.8.7 [V5-stable] 2012-01-17 - bugfix: instabilities when using RFC5424 header fields Thanks to Kaiwang Chen for the patch @@ -294,6 +413,7 @@ Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 Version 5.6.5 [V5-STABLE] (rgerhards), 2011-03-22 - bugfix: failover did not work correctly if repeated msg reduction was on affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 - bugfix: omlibdbi did not use password from rsyslog.con closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203 - bugfix(kind of): tell users that config graph can currently not be diff --git a/Makefile.am b/Makefile.am index 48eb26ce..7cc1b65c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -170,6 +170,10 @@ if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif +if ENABLE_ELASTICSEARCH +SUBDIRS += plugins/omelasticsearch +endif + if ENABLE_MMSNMPTRAPD SUBDIRS += plugins/mmsnmptrapd endif @@ -246,5 +250,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-imtemplate \ --enable-omtemplate \ --enable-mmsnmptrapd \ + --enable-elasticsearch \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 @@ -321,17 +321,17 @@ rsRetVal actionConstructFinalize(action_t *pThis) { DEFiRet; - uchar pszAName[64]; /* friendly name of our queue */ + uchar pszAName[64]; /* friendly name of our action */ ASSERT(pThis != NULL); /* generate a friendly name for us action stats */ - if(pThis->pszName == NULL) { + if(pThis->pszName == NULL) { snprintf((char*) pszAName, sizeof(pszAName)/sizeof(uchar), "action %d", iActionNbr); - } else { + } else { ustrncpy(pszAName, pThis->pszName, sizeof(pszAName)); pszAName[63] = '\0'; /* to be on the save side */ - } + } /* support statistics gathering */ CHKiRet(statsobj.Construct(&pThis->statsobj)); @@ -348,9 +348,15 @@ actionConstructFinalize(action_t *pThis) CHKiRet(statsobj.ConstructFinalize(pThis->statsobj)); /* create our queue */ - /* find a name for our queue */ - snprintf((char*) pszAName, sizeof(pszAName)/sizeof(uchar), "action %d queue", iActionNbr); + /* generate a friendly name for the queue */ + if(pThis->pszName == NULL) { + snprintf((char*) pszAName, sizeof(pszAName)/sizeof(uchar), "action %d queue", + iActionNbr); + } else { + ustrncpy(pszAName, pThis->pszName, sizeof(pszAName)); + pszAName[63] = '\0'; /* to be on the save side */ + } /* now check if we can run the action in "firehose mode" during stage one of * its processing (that is before messages are enqueued into the action q). * This is only possible if some features, which require strict sequence, are @@ -1793,7 +1799,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques } /* check required template options */ if( (iTplOpts & OMSR_RQD_TPL_OPT_SQL) - && (pAction->ppTpl[i]->optFormatForSQL == 0)) { + && (pAction->ppTpl[i]->optFormatEscape == 0)) { errno = 0; errmsg.LogError(0, RS_RET_RQD_TPLOPT_MISSING, "Action disabled. To use this action, you have to specify " "the SQL or stdSQL option in your template!\n"); diff --git a/configure.ac b/configure.ac index c6d775af..32029409 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.8.6-newstats6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.9.4],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -37,6 +37,7 @@ PKG_PROG_PKG_CONFIG case "${host}" in *-*-linux*) + AC_DEFINE([OS_LINUX], [1], [Indicator for a Linux OS]) os_type="linux" ;; *-*-*darwin*|*-*-freebsd*|*-*-netbsd*|*-*-openbsd*) @@ -118,6 +119,8 @@ AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname get # let me know! -- rgerhards, 2010-10-06 AC_CHECK_DECL([SCM_CREDENTIALS], [AC_DEFINE(HAVE_SCM_CREDENTIALS, [1], [set define])], [], [#include <sys/types.h> #include <sys/socket.h>]) +AC_CHECK_DECL([SO_TIMESTAMP], [AC_DEFINE(HAVE_SO_TIMESTAMP, [1], [set define])], [], [#include <sys/types.h> +#include <sys/socket.h>]) # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) @@ -669,6 +672,40 @@ AC_SUBST(SNMP_CFLAGS) AC_SUBST(SNMP_LIBS) +# elasticsearch support +AC_ARG_ENABLE(elasticsearch, + [AS_HELP_STRING([--enable-elasticsearch],[Enable elasticsearch output module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_elasticsearch="yes" ;; + no) enable_elasticsearch="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-elasticsearch) ;; + esac], + [enable_elasticsearch=no] +) +if test "x$enable_elasticsearch" = "xyes"; then + AC_CHECK_PROG( + [HAVE_CURL_CONFIG], + [curl-config], + [yes],,, + ) + if test "x${HAVE_CURL_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([curl-config not found in PATH]) + fi + AC_CHECK_LIB( + [curl], + [curl_global_init], + [CURL_CFLAGS="`curl-config --cflags`" + CURL_LIBS="`curl-config --libs`" + ], + [AC_MSG_FAILURE([curl library is missing])], + [`curl-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ELASTICSEARCH, test x$enable_elasticsearch = xyes) +AC_SUBST(CURL_CFLAGS) +AC_SUBST(CURL_LIBS) + + # GnuTLS support AC_ARG_ENABLE(gnutls, [AS_HELP_STRING([--enable-gnutls],[Enable GNU TLS support @<:@default=no@:>@])], @@ -1205,6 +1242,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ plugins/omudpspoof/Makefile \ + plugins/omelasticsearch/Makefile \ plugins/sm_cust_bindcdr/Makefile \ plugins/mmsnmptrapd/Makefile \ plugins/cust1/Makefile \ @@ -1239,6 +1277,7 @@ echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" echo " omhdfs module will be compiled: $enable_omhdfs" +echo " omelasticsearch module will be compiled: $enable_elasticsearch" echo " omruleset module will be compiled: $enable_omruleset" echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" diff --git a/doc/imfile.html b/doc/imfile.html index c44171df..7961729b 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -100,9 +100,16 @@ performance, especially when set to a low value. Frequently writing the state file is very time consuming. <li><b>$InputFileReadMode</b> [mode]</b><br> Available in 5.7.5+ +<li><b>$InputFileMaxLinesAtOnce</b> [number]</b><br> +Available in 5.9.0+ <br> -Mode to be used when reading lines. 0 (the default) means that each line is forwarded -as its own log message. +This is useful if multiple files need to be monitored. If set to 0, each file +will be fully processed and then processing switches to the next file +(this was the default in previous versions). If it is set, a maximum of +[number] lines is processed in sequence for each file, and then the file is +switched. This provides a kind of mutiplexing the load of multiple files and +probably leads to a more natural distribution of events when multiple busy files +are monitored. The default is 10240. <li>$InputFileBindRuleset <ruleset><br> Available in 5.7.5+, 6.1.5+ Binds the listener to a specific <a href="multi_ruleset.html">ruleset</a>.</li> diff --git a/doc/imptcp.html b/doc/imptcp.html index d4228185..386e691a 100644 --- a/doc/imptcp.html +++ b/doc/imptcp.html @@ -45,7 +45,25 @@ can be found at the <a href="http://www.rsyslog.com/Article321.phtml">Cisco tcp page. <li>$InputPTCPServerNotifyOnConnectionClose [on/<b>off</b>]<br> instructs imptcp to emit a message if the remote peer closes a connection.<br> -<li>$InputPTCPServerRun <port><br> +<li><b>$InputPTCPServerKeepAlive</b> <on/<b>off</b>><br> +enable of disable keep-alive packets at the tcp socket layer. The default is +to disable them.</li> +<li><b>$InputPTCPServerKeepAlive_probes</b> <number><br> +The number of unacknowledged probes to send before considering the connection dead and notifying the application layer. +The default, 0, means that the operating system defaults are used. This has only +effect if keep-alive is enabled. The functionality may not be available on +all platforms. +<li><b>$InputPTCPServerKeepAlive_intvl</b> <number><br> +The interval between subsequential keepalive probes, regardless of what the connection has exchanged in the meantime. +The default, 0, means that the operating system defaults are used. This has only +effect if keep-alive is enabled. The functionality may not be available on +all platforms. +<li><b>$InputPTCPServerKeepAlive_time</b> <number><br> +The interval between the last data packet sent (simple ACKs are not considered data) and the first keepalive probe; after the connection is marked to need keepalive, this counter is not used any further. +The default, 0, means that the operating system defaults are used. This has only +effect if keep-alive is enabled. The functionality may not be available on +all platforms. +<li><b>$InputPTCPServerRun</b> <port><br> Starts a TCP server on selected port</li> <li>$InputPTCPServerInputName <name><br> Sets a name for the inputname property. If no name is set "imptcp" is used by default. Setting a diff --git a/doc/imtcp.html b/doc/imtcp.html index 422bbd55..7653f601 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -55,8 +55,20 @@ so be prepared to wrangle with that! instructs imtcp to emit a message if the remote peer closes a connection.<br> <b>Important:</b> This directive is global to all listeners and must be given right after loading imtcp, otherwise it may have no effect.</li> +<li><b>$InputTCPServerKeepAlive</b> <on/<b>off</b>><br> +enable of disable keep-alive packets at the tcp socket layer. The default is +to disable them.</li> <li><b>$InputTCPServerRun</b> <port><br> Starts a TCP server on selected port</li> +<li><b>$InputTCPFlowControl</b> <<b>on</b>/off><br> +This setting specifies whether some message flow control shall be exercised on the +related TCP input. If set to on, messages are handled as "light delayable", which means +the sender is throttled a bit when the queue becomes near-full. This is done in order +to preserve some queue space for inputs that can not throttle (like UDP), but it +may have some undesired effect in some configurations. Still, we consider this as +a useful setting and thus it is the default. To turn the handling off, simply +configure that explicitely. +</li> <li><b>$InputTCPMaxListeners</b> <number><br> Sets the maximum number of listeners (server ports) supported. Default is 20. This must be set before the first $InputTCPServerRun directive.</li> <li><b>$InputTCPMaxSessions</b> <number><br> Sets the maximum number of sessions supported. Default is 200. This must be set before the first $InputTCPServerRun directive</li> diff --git a/doc/imuxsock.html b/doc/imuxsock.html index ee5db22d..734ae889 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -49,6 +49,15 @@ are places as quickly as possible into the processing queues. If you would like flow control, you need to enable it via the $SystemLogSocketFlowControl and $InputUnixListenSocketFlowControl config directives. Just make sure you thought about the implications. Note that for many systems, turning on flow control does not hurt. +<p>Starting with rsyslog 5.9.4, +<b><a href="http://www.rsyslog.com/what-are-trusted-properties/">trusted syslog properties</a> +are available</b>. These require a recent enough Linux Kernel and access to the /proc file +system. In other words, this may not work on all platforms and may not work fully when +privileges are dropped (depending on how they are dropped). Note that trusted properties +can be very useful, but also typically cause the message to grow rather large. Also, the +format of log messages is obviously changed by adding the trusted properties at the end. +For these reasons, the feature is <b>not enabled by default</b>. If you want to use it, +you must turn it on (via $SystemLogSocketAnnotate and $InputUnixListenSocketAnnotate). <p><b>Configuration Directives</b>:</p> <ul> <li><b>$InputUnixListenSocketIgnoreMsgTimestamp</b> [<b>on</b>/off] @@ -56,7 +65,10 @@ the implications. Note that for many systems, turning on flow control does not h <li><b>$InputUnixListenSocketFlowControl</b> [on/<b>off</b>] - specifies if flow control should be applied to the next socket.</li> <li><b>$IMUXSockRateLimitInterval</b> [number] - specifies the rate-limiting -interval in seconds. Default value is 5 seconds. Set it to 0 to turn rate limiting off. +interval in seconds. Default value is 0, which turns off rate limiting. Set it to a number +of seconds (5 recommended) to activate rate-limiting. The default of 0 has been choosen in 5.9.6+, +as people experienced problems with this feature activated by default. Now it needs an +explicit opt-in by setting this parameter. </li> <li><b>$IMUXSockRateLimitBurst</b> [number] - specifies the rate-limiting burst in number of messages. Default is 200. @@ -68,9 +80,18 @@ messages that shall be rate-limited. be obtained from the log socket itself. If so, the TAG part of the message is rewritten. It is recommended to turn this option on, but the default is "off" to keep compatible with earlier versions of rsyslog. This option was introduced in 5.7.0.</li> +<li><b>$InputUnixListenSocketUseSysTimeStamp</b> [<b>on</b>/off] instructs imuxsock +to obtain message time from the system (via control messages) insted of using time +recorded inside the message. This may be most useful in combination with systemd. Note: +this option was introduced with version 5.9.1. Due to the usefulness of it, we +decided to enable it by default. As such, 5.9.1 and above behave slightly different +than previous versions. However, we do not see how this could negatively affect +existing environments.<br> <li><b>$SystemLogSocketIgnoreMsgTimestamp</b> [<b>on</b>/off]<br> Ignore timestamps included in the messages, applies to messages received via the system log socket.</li> -<li><b>$OmitLocalLogging</b> (imuxsock) [on/<b>off</b>] -- former -o option</li> +<li><b>$OmitLocalLogging</b> (imuxsock) [on/<b>off</b>] -- former -o option; +do NOT listen for the local log socket. This is most useful if you run multiple +instances of rsyslogd where only one shall handle the system log socket.</li> <li><b>$SystemLogSocketName</b> <name-of-socket> -- former -p option</li> <li><b>$SystemLogFlowControl</b> [on/<b>off</b>] - specifies if flow control should be applied to the system log socket.</li> @@ -87,6 +108,7 @@ burst in number of messages. Default is 200. <li><b>$SystemLogRateLimitSeverity</b> [numerical severity] - specifies the severity of messages that shall be rate-limited. </li> +<li><b>$SystemLogUseSysTimeStamp</b> [<b>on</b>/off] the same as $InputUnixListenSocketUseSysTimeStamp, but for the system log socket. <li><b>$InputUnixListenSocketCreatePath</b> [on/<b>off</b>] - create directories in the socket path if they do not already exist. They are created with 0755 permissions with the owner being the process under which rsyslogd runs. The default is not to create directories. Keep in mind, though, that rsyslogd always @@ -105,7 +127,12 @@ shall be used inside messages taken from the <b>next</b> $AddUnixListenSocket so the hostname must be specified before the $AddUnixListenSocket configuration directive, and it will only affect the next one and then automatically be reset. This functionality is provided so that the local hostname can be overridden in cases where that is desired.</li> +<li><b>$InputUnixListenSocketAnnotate</b> <on/<b>off</b>> turn on annotation/trusted +properties for the non-system log socket in question.</li> +<li><b>$SystemLogSocketAnnotate</b> <on/<b>off</b>> turn on annotation/trusted +properties for the system log socket.</li> </ul> + <b>Caveats/Known Bugs:</b><br> <ul> <li>There is a compile-time limit of 50 concurrent sockets. If you need more, you need to @@ -141,17 +168,21 @@ $InputUnixListenSocketHostName /var/run/sshd/dev/log </textarea> <p>The following sample is used to turn off input rate limiting on the system log socket. -<textarea rows="6" cols="70">$ModLoad imuxsock # needs to be done just once +<textarea rows="4" cols="70">$ModLoad imuxsock # needs to be done just once $SystemLogRateLimitInterval 0 # turn off rate limiting </textarea> +<p>The following sample is used activate message annotation and thus trusted properties +on the system log socket. +<textarea rows="4" cols="70">$ModLoad imuxsock # needs to be done just once + +$SystemLogSocketAnnotate on +</textarea> <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the -<a href="http://www.rsyslog.com/">rsyslog</a> -project.<br> -Copyright © 2008-2010 by <a href="http://www.gerhards.net/rainer">Rainer -Gerhards</a> and +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> </body></html> diff --git a/doc/manual.html b/doc/manual.html index 010ccab6..fdeb2980 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 5.8.7 (v5-stable branch) of rsyslog.</b> +<p><b>This documentation is for version 5.9.4 (stable branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might diff --git a/doc/mmsnmptrapd.html b/doc/mmsnmptrapd.html index e69bc241..699049d3 100644 --- a/doc/mmsnmptrapd.html +++ b/doc/mmsnmptrapd.html @@ -51,8 +51,11 @@ to control output modules are also available to mmsnmptrapd. <ul> <li><b>$mmsnmptrapdTag</b> [tagname]<br> tells the module which start string inside the tag to look for. The default is -"snmptrap/" -<li><b>$mmsnmptrapdSevertiyMapping</b> [severtiymap]<br> +"snmptrapd". Note that a slash is automatically added to this tag when it comes to +matching incoming messages. It MUST not be given, except if two slashes are required +for whatever reasons (so "tag/" results in a check for "tag//" at the start of +the tag field). +<li><b>$mmsnmptrapdSeverityMapping</b> [severtiymap]<br> This specifies the severity mapping table. It needs to be specified as a list. Note that due to the current config system <b>no whitespace</b> is supported inside the list, so be sure not to use any whitespace inside it.<br> @@ -76,7 +79,7 @@ severities. The default tag is used.<br> # ... other module loads and listener setup ... *.* /path/to/file/with/orignalMessage # this file receives *un*modified messages $mmsnmptrapdSeverityMapping warning/4,error/3 -*.* ::mmsnmptrapd: # *now* message is modified +*.* :mmsnmptrapd: # *now* message is modified *.* /path/to/file/with/modifiedMessage # this file receives modified messages # ... rest of config ... </textarea> diff --git a/doc/rsconf1_omfileforcechown.html b/doc/rsconf1_omfileforcechown.html index 7415a6f6..a680810b 100644 --- a/doc/rsconf1_omfileforcechown.html +++ b/doc/rsconf1_omfileforcechown.html @@ -8,7 +8,10 @@ <h2>$omfileForceChown</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Parameter Values:</b> boolean (on/off, yes/no)</p> -<p><b>Available since:</b> 4.7.0+, 5.3.0+</p> +<p><b>Available:</b> 4.7.0+, 5.3.0-5.8.x, <b>NOT</b> available in 5.9.x or higher</p> +<p><b>Note: this directive has been removed and is no longer available. The +documentation is currently being retained for historical reaons.</b> Expect +it to go away at some later stage as well. <p><b>Default:</b> off</p> <p><b>Description:</b></p> <p>Forces rsyslogd to change the ownership for output files that already exist. Please note diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 21786a7f..b254f366 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -143,6 +143,7 @@ our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a <li><a href="rsconf1_escape8bitcharsonreceive.html">$Escape8BitCharactersOnReceive</a></li> <li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> <li><b>$EscapeControlCharactersOnReceive</b> [<b>on</b>|off] - escape USASCII HT character</li> +<li>$SpaceLFOnReceive [on/<b>off</b>] - instructs rsyslogd to replace LF with spaces during message reception (sysklogd compatibility aid)</li> <li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> <li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li> <li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li> diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index 23a02049..bd0b3253 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -146,6 +146,10 @@ with high-precision timestamps and timezone information</li> useful if you send messages to other syslogd's or rsyslogd below version 3.12.5.</li> +<li><span style="font-weight: bold;">RSYSLOG_SysklogdFileFormat</span> +- sysklogd compatible log file format. If used with options: $SpaceLFOnReceive on; +$EscapeControlCharactersOnReceive off; $DropTrailingLFOnReception off, +the log format will conform to sysklogd log format.</li> <li><span style="font-weight: bold;">RSYSLOG_ForwardFormat</span> - a new high-precision forwarding format very similar to the traditional one, but with high-precision timestamps and timezone diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index ba8318df..632bf390 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -63,6 +63,7 @@ DEFobjCurrIf(strm) DEFobjCurrIf(prop) DEFobjCurrIf(ruleset) +#define NUM_MULTISUB 1024 /* max number of submits -- TODO: make configurable */ typedef struct fileInfo_s { uchar *pszFileName; uchar *pszTag; @@ -70,11 +71,13 @@ typedef struct fileInfo_s { uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; + int maxLinesAtOnce; int nRecords; /**< How many records did we process before persisting the stream? */ int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ int readMode; /* which mode to use in ReadMulteLine call? */ ruleset_t *pRuleset; /* ruleset to bind listener to (use system default if unspecified) */ + multi_submit_t multiSub; } fileInfo_t; @@ -90,6 +93,7 @@ static int iPersistStateInterval = 0; /* how often if state file to be persisted static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ static int readMode = 0; /* mode to use for ReadMultiLine call */ +static int maxLinesAtOnce = 10240; /* how many lines to process in a row? */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ static int iFilPtr = 0; /* number of files to be monitored; pointer to next free spot during config */ @@ -121,7 +125,9 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); MsgSetRuleset(pMsg, pInfo->pRuleset); - CHKiRet(submitMsg(pMsg)); + pInfo->multiSub.ppMsgs[pInfo->multiSub.nElem++] = pMsg; + if(pInfo->multiSub.nElem == pInfo->multiSub.maxElem) + CHKiRet(multiSubmitMsg(&pInfo->multiSub)); finalize_it: RETiRet; } @@ -205,6 +211,7 @@ static void pollFileCancelCleanup(void *pArg) static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) { cstr_t *pCStr = NULL; + int nProcessed = 0; DEFiRet; ASSERT(pbHadFileData != NULL); @@ -219,7 +226,10 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) /* loop below will be exited when strmReadLine() returns EOF */ while(glbl.GetGlobalInputTermState() == 0) { + if(pThis->maxLinesAtOnce != 0 && nProcessed >= pThis->maxLinesAtOnce) + break; CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr, pThis->readMode)); + ++nProcessed; *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ @@ -230,6 +240,10 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) } finalize_it: + if(pThis->multiSub.nElem > 0) { + /* submit everything that was not yet submitted */ + CHKiRet(multiSubmitMsg(&pThis->multiSub)); + } ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ /* Note: the problem above is that pthread:cleanup_pop() is a macro which * evaluates to something like "} while(0);". So the code would become @@ -474,6 +488,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iSeverity = 5; /* notice, as of rfc 3164 */ readMode = 0; pBindRuleset = NULL; + maxLinesAtOnce = 10240; RETiRet; } @@ -512,8 +527,12 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->pszStateFile = (uchar*) strdup((char*) pszStateFile); } + CHKmalloc(pThis->multiSub.ppMsgs = MALLOC(NUM_MULTISUB * sizeof(msg_t*))); + pThis->multiSub.maxElem = NUM_MULTISUB; + pThis->multiSub.nElem = 0; pThis->iSeverity = iSeverity; pThis->iFacility = iFacility; + pThis->maxLinesAtOnce = maxLinesAtOnce; pThis->iPersistStateInterval = iPersistStateInterval; pThis->nRecords = 0; pThis->readMode = readMode; @@ -591,6 +610,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iPollInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilereadmode", 0, eCmdHdlrInt, NULL, &readMode, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilemaxlinesatonce", 0, eCmdHdlrSize, + NULL, &maxLinesAtOnce, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilebindruleset", 0, eCmdHdlrGetWord, diff --git a/plugins/imklog/Makefile.am b/plugins/imklog/Makefile.am index 5d4d0465..7d0d37c9 100644 --- a/plugins/imklog/Makefile.am +++ b/plugins/imklog/Makefile.am @@ -1,5 +1,4 @@ pkglib_LTLIBRARIES = imklog.la - imklog_la_SOURCES = imklog.c imklog.h # select klog "driver" @@ -8,7 +7,7 @@ imklog_la_SOURCES += bsd.c endif if ENABLE_IMKLOG_LINUX -imklog_la_SOURCES += linux.c module.h ksym.c ksyms.h ksym_mod.c +imklog_la_SOURCES += bsd.c endif imklog_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index 0a4c7cd4..eaf8e5ca 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -1,69 +1,30 @@ -/* klog for BSD, based on the FreeBSD syslogd implementation. +/* combined imklog driver for BSD and Linux * * This contains OS-specific functionality to read the BSD - * kernel log. For a general overview, see head comment in - * imklog.c. + * or Linux kernel log. For a general overview, see head comment in + * imklog.c. This started out as the BSD-specific drivers, but it + * turned out that on modern Linux the implementation details + * are very small, and so we use a single driver for both OS's with + * a little help of conditional compilation. * - * Copyright (C) 2008 by Rainer Gerhards for the modifications of - * the original FreeBSD sources. - * - * I would like to express my gratitude to those folks which - * layed an important foundation for rsyslog to build on. + * Copyright 2008-2012 Adiscon GmbH * * This file is part of rsyslog. * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - * - * This file is based on earlier work included in the FreeBSD sources. We - * integrated it into the rsyslog project. The copyright below applies, and - * I also reproduce the original license under which we aquired the code: - * - * Copyright (c) 1983, 1988, 1993, 1994 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * If you would like to use the code under the BSD license, you should - * aquire your own copy of BSD's syslogd, from which we have taken it. The - * code in this file is modified and may only be used under the terms of - * the GPLv3+ as specified above. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - #ifdef HAVE_CONFIG_H # include "config.h" #endif @@ -72,18 +33,124 @@ #include <fcntl.h> #include <errno.h> #include <string.h> +#include <ctype.h> +#ifdef OS_LINUX +# include <sys/klog.h> +#endif #include "rsyslog.h" -#include "imklog.h" +#include "srUtils.h" #include "debug.h" +#include "imklog.h" /* globals */ -static int fklog = -1; /* /dev/klog */ +static int fklog = -1; /* kernel log fd */ #ifndef _PATH_KLOG -# define _PATH_KLOG "/dev/klog" +# ifdef OS_LINUX +# define _PATH_KLOG "/proc/kmsg" +# else +# define _PATH_KLOG "/dev/klog" +# endif #endif + +#ifdef OS_LINUX +/* submit a message to imklog Syslog() API. In this function, we check if + * a kernel timestamp is present and, if so, extract and strip it. + * Note: this is an extra processing step. We should revisit the whole + * idea in v6 and remove all that old stuff that we do not longer need + * (like symbol resolution). <-- TODO + * Note that this is heavily Linux specific and thus is not compiled or + * used for BSD. + * Special thanks to Lennart Poettering for suggesting on how to convert + * the kernel timestamp to a realtime timestamp. This method depends on + * the fact the the kernel timestamp is written using the monotonic clock. + * Shall that change (very unlikely), this code must be changed as well. Note + * that due to the way we generate the delta, we are unable to write the + * absolutely correct timestamp (system call overhead of the clock calls + * prevents us from doing so). However, the difference is very minor. + * rgerhards, 2011-06-24 + */ +static void +submitSyslog(int pri, uchar *buf) +{ + long secs; + long nsecs; + long secOffs; + long nsecOffs; + unsigned i; + unsigned bufsize; + struct timespec monotonic, realtime; + struct timeval tv; + struct timeval *tp = NULL; + + if(buf[3] != '[') + goto done; + DBGPRINTF("imklog: kernel timestamp detected, extracting it\n"); + + /* we now try to parse the timestamp. iff it parses, we assume + * it is a timestamp. Otherwise we know for sure it is no ts ;) + */ + i = 4; /* first digit after '[' */ + secs = 0; + while(buf[i] && isdigit(buf[i])) { + secs = secs * 10 + buf[i] - '0'; + ++i; + } + if(buf[i] != '.') { + DBGPRINTF("no dot --> no kernel timestamp\n"); + goto done; /* no TS! */ + } + + ++i; /* skip dot */ + nsecs = 0; + while(buf[i] && isdigit(buf[i])) { + nsecs = nsecs * 10 + buf[i] - '0'; + ++i; + } + if(buf[i] != ']') { + DBGPRINTF("no trailing ']' --> no kernel timestamp\n"); + goto done; /* no TS! */ + } + ++i; /* skip ']' */ + + /* we have a timestamp */ + DBGPRINTF("kernel timestamp is %ld %ld\n", secs, nsecs); + bufsize= strlen((char*)buf); + memcpy(buf+3, buf+i, bufsize - i + 1); + + clock_gettime(CLOCK_MONOTONIC, &monotonic); + clock_gettime(CLOCK_REALTIME, &realtime); + secOffs = realtime.tv_sec - monotonic.tv_sec; + nsecOffs = realtime.tv_nsec - monotonic.tv_nsec; + if(nsecOffs < 0) { + secOffs--; + nsecOffs += 1000000000l; + } + + nsecs +=nsecOffs; + if(nsecs > 999999999l) { + secs++; + nsecs -= 1000000000l; + } + secs += secOffs; + tv.tv_sec = secs; + tv.tv_usec = nsecs / 1000; + tp = &tv; + +done: + Syslog(pri, buf, tp); +} +#else /* now comes the BSD "code" (just a shim) */ +static void +submitSyslog(int pri, uchar *buf) +{ + Syslog(pri, buf, NULL); +} +#endif /* #ifdef LINUX */ + + static uchar *GetPath(void) { return pszPath ? pszPath : (uchar*) _PATH_KLOG; @@ -95,23 +162,36 @@ static uchar *GetPath(void) rsRetVal klogWillRun(void) { + char errmsg[2048]; + int r; DEFiRet; fklog = open((char*)GetPath(), O_RDONLY, 0); if (fklog < 0) { - dbgprintf("can't open %s (%d)\n", GetPath(), errno); - iRet = RS_RET_ERR; // TODO: better error code + imklogLogIntMsg(RS_RET_ERR_OPEN_KLOG, "imklog: cannot open kernel log(%s): %s.", + GetPath(), rs_strerror_r(errno, errmsg, sizeof(errmsg))); + ABORT_FINALIZE(RS_RET_ERR_OPEN_KLOG); + } + +# ifdef OS_LINUX + /* Set level of kernel console messaging.. */ + if(console_log_level != -1) { + r = klogctl(8, NULL, console_log_level); + if(r != 0) { + imklogLogIntMsg(LOG_WARNING, "imklog: cannot set console log level: %s", + rs_strerror_r(errno, errmsg, sizeof(errmsg))); + /* make sure we do not try to re-set! */ + console_log_level = -1; + } } +# endif /* #ifdef OS_LINUX */ +finalize_it: RETiRet; } -/* Read /dev/klog while data are available, split into lines. - * Contrary to standard BSD syslogd, we do a blocking read. We can - * afford this as imklog is running on its own threads. So if we have - * a single file, it really doesn't matter if we wait inside a 1-file - * select or the read() directly. +/* Read kernel log while data are available, split into lines. */ static void readklog(void) @@ -119,13 +199,14 @@ readklog(void) char *p, *q; int len, i; int iMaxLine; - uchar bufRcv[4096+1]; + uchar bufRcv[128*1024+1]; + char errmsg[2048]; uchar *pRcv = NULL; /* receive buffer */ iMaxLine = klog_getMaxLine(); - /* we optimize performance: if iMaxLine is below 4K (which it is in almost all - * cases, we use a fixed buffer on the stack. Only if it is higher, heap memory + /* we optimize performance: if iMaxLine is below our fixed size buffer (which + * usually is sufficiently large), we use this buffer. if it is higher, heap memory * is used. We could use alloca() to achive a similar aspect, but there are so * many issues with alloca() that I do not want to take that route. * rgerhards, 2008-09-02 @@ -139,15 +220,15 @@ readklog(void) len = 0; for (;;) { - dbgprintf("----------imklog(BSD) waiting for kernel log line\n"); + dbgprintf("imklog(BSD/Linux) waiting for kernel log line\n"); i = read(fklog, pRcv + len, iMaxLine - len); if (i > 0) { pRcv[i + len] = '\0'; } else { if (i < 0 && errno != EINTR && errno != EAGAIN) { imklogLogIntMsg(LOG_ERR, - "imklog error %d reading kernel log - shutting down imklog", - errno); + "imklog: error reading kernel log - shutting down: %s", + rs_strerror_r(errno, errmsg, sizeof(errmsg))); fklog = -1; } break; @@ -155,18 +236,18 @@ readklog(void) for (p = (char*)pRcv; (q = strchr(p, '\n')) != NULL; p = q + 1) { *q = '\0'; - Syslog(LOG_INFO, (uchar*) p); + submitSyslog(LOG_INFO, (uchar*) p); } len = strlen(p); if (len >= iMaxLine - 1) { - Syslog(LOG_INFO, (uchar*)p); + submitSyslog(LOG_INFO, (uchar*)p); len = 0; } - if (len > 0) + if(len > 0) memmove(pRcv, p, len + 1); } if (len > 0) - Syslog(LOG_INFO, pRcv); + submitSyslog(LOG_INFO, pRcv); if(pRcv != NULL && (size_t) iMaxLine >= sizeof(bufRcv) - 1) free(pRcv); @@ -181,6 +262,11 @@ rsRetVal klogAfterRun(void) DEFiRet; if(fklog != -1) close(fklog); +# ifdef OS_LINUX + /* Turn on logging of messages to console, but only if a log level was speficied */ + if(console_log_level != -1) + klogctl(7, NULL, 0); +# endif RETiRet; } diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 69c8cd1a..40249273 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -18,7 +18,10 @@ * Please note that this file replaces the klogd daemon that was * also present in pre-v3 versions of rsyslog. * - * Copyright (C) 2008, 2009 by Rainer Gerhards and Adiscon GmbH + * To test under Linux: + * echo test1 > /dev/kmsg + * + * Copyright (C) 2008-2011 by Rainer Gerhards and Adiscon GmbH * * This file is part of rsyslog. * @@ -93,15 +96,21 @@ static prop_t *pLocalHostIP = NULL; /* a pseudo-constant propterty for 127.0.0.1 * rgerhards, 2008-04-12 */ static rsRetVal -enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) +enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity, struct timeval *tp) { - DEFiRet; + struct syslogTime st; msg_t *pMsg; + DEFiRet; assert(msg != NULL); assert(pszTag != NULL); - CHKiRet(msgConstruct(&pMsg)); + if(tp == NULL) { + CHKiRet(msgConstruct(&pMsg)); + } else { + datetime.timeval2syslogTime(tp, &st); + CHKiRet(msgConstructWithTime(&pMsg, &st, tp->tv_sec)); + } MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, pInputName); MsgSetRawMsgWOSize(pMsg, (char*)msg); @@ -174,31 +183,47 @@ rsRetVal imklogLogIntMsg(int priority, char *fmt, ...) va_end(ap); iRet = enqMsg((uchar*)pLogMsg, (uchar*) ((iFacilIntMsg == LOG_KERN) ? "kernel:" : "imklog:"), - iFacilIntMsg, LOG_PRI(priority)); + iFacilIntMsg, LOG_PRI(priority), NULL); RETiRet; } -/* log a kernel message +/* log a kernel message. If tp is non-NULL, it contains the message creation + * time to use. * rgerhards, 2008-04-14 */ -rsRetVal Syslog(int priority, uchar *pMsg) +rsRetVal Syslog(int priority, uchar *pMsg, struct timeval *tp) { - DEFiRet; + int pri = -1; rsRetVal localRet; + DEFiRet; - /* Output using syslog */ - localRet = parsePRI(&pMsg, &priority); - if(localRet != RS_RET_INVALID_PRI && localRet != RS_RET_OK) - FINALIZE; + /* then check if we have two PRIs. This can happen in case of systemd, + * in which case the second PRI is the right one. + */ + if(pMsg[3] == '<' || (pMsg[3] == ' ' && pMsg[4] == '<')) { /* could be a pri... */ + uchar *pMsgTmp = pMsg + ((pMsg[3] == '<') ? 3 : 4); + localRet = parsePRI(&pMsgTmp, &pri); + if(localRet == RS_RET_OK && pri >= 8 && pri <= 192) { + /* *this* is our PRI */ + DBGPRINTF("imklog detected secondary PRI(%d) in klog msg\n", pri); + pMsg = pMsgTmp; + priority = pri; + } + } + if(pri == -1) { + localRet = parsePRI(&pMsg, &priority); + if(localRet != RS_RET_INVALID_PRI && localRet != RS_RET_OK) + FINALIZE; + } /* if we don't get the pri, we use whatever we were supplied */ /* ignore non-kernel messages if not permitted */ if(bPermitNonKernel == 0 && LOG_FAC(priority) != LOG_KERN) FINALIZE; /* silently ignore */ - iRet = enqMsg((uchar*)pMsg, (uchar*) "kernel:", LOG_FAC(priority), LOG_PRI(priority)); + iRet = enqMsg((uchar*)pMsg, (uchar*) "kernel:", LOG_FAC(priority), LOG_PRI(priority), tp); finalize_it: RETiRet; diff --git a/plugins/imklog/imklog.h b/plugins/imklog/imklog.h index c183026d..39bdeb5c 100644 --- a/plugins/imklog/imklog.h +++ b/plugins/imklog/imklog.h @@ -5,7 +5,7 @@ * Major change: 2008-04-09: switched to a driver interface for * several platforms * - * Copyright 2007-2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2011 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -56,7 +56,7 @@ extern uchar *pszPath; /* the functions below may be called by the drivers */ rsRetVal imklogLogIntMsg(int priority, char *fmt, ...) __attribute__((format(printf,2, 3))); -rsRetVal Syslog(int priority, uchar *msg); +rsRetVal Syslog(int priority, uchar *msg, struct timeval *tp); /* prototypes */ extern int klog_getMaxLine(void); /* work-around for klog drivers to get configured max line size */ diff --git a/plugins/imklog/ksym.c b/plugins/imklog/ksym.c deleted file mode 100644 index ebaec011..00000000 --- a/plugins/imklog/ksym.c +++ /dev/null @@ -1,832 +0,0 @@ -/* ksym.c - functions for kernel address->symbol translation - * Copyright (c) 1995, 1996 Dr. G.W. Wettstein <greg@wind.rmcc.com> - * Copyright (c) 1996 Enjellic Systems Development - * Copyright (c) 1998-2007 Martin Schulze <joey@infodrom.org> - * Copyright (C) 2007-2008 Rainer Gerhards <rgerhards@adiscon.com> - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. -*/ - -/* - * This file contains functions which handle the translation of kernel - * numeric addresses into symbols for the klogd utility. - * - * Sat Oct 28 09:00:14 CDT 1995: Dr. Wettstein - * Initial Version. - * - * Fri Nov 24 12:50:52 CST 1995: Dr. Wettstein - * Added VERBOSE_DEBUGGING define to make debugging output more - * manageable. - * - * Added support for verification of the loaded kernel symbols. If - * no version information can be be found in the mapfile a warning - * message is issued but translation will still take place. This - * will be the default case if kernel versions < 1.3.43 are used. - * - * If the symbols in the mapfile are of the same version as the kernel - * that is running an informative message is issued. If the symbols - * in the mapfile do not match the current kernel version a warning - * message is issued and translation is disabled. - * - * Wed Dec 6 16:14:11 CST 1995: Dr. Wettstein - * Added /boot/System.map to the list of symbol maps to search for. - * Also made this map the first item in the search list. I am open - * to CONSTRUCTIVE suggestions for any additions or corrections to - * the list of symbol maps to search for. Be forewarned that the - * list in use is the consensus agreement between myself, Linus and - * some package distributers. It is a given that no list will suit - * everyone's taste. If you have rabid concerns about the list - * please feel free to edit the system_maps array and compile your - * own binaries. - * - * Added support for searching of the list of symbol maps. This - * allows support for access to multiple symbol maps. The theory - * behind this is that a production kernel may have a system map in - * /boot/System.map. If a test kernel is booted this system map - * would be skipped in favor of one found in /usr/src/linux. - * - * Thu Jan 18 11:18:31 CST 1996: Dr. Wettstein - * Added patch from beta-testers to allow for reading of both - * ELF and a.out map files. - * - * Wed Aug 21 09:15:49 CDT 1996: Dr. Wettstein - * Reloading of kernel module symbols is now turned on by the - * SetParanoiaLevel function. The default behavior is to NOT reload - * the kernel module symbols when a protection fault is detected. - * - * Added support for freeing of the current kernel module symbols. - * This was necessary to support reloading of the kernel module symbols. - * - * When a matching static symbol table is loaded the kernel version - * number is printed. - * - * Mon Jun 9 17:12:42 CST 1997: Martin Schulze - * Added #1 and #2 to some error messages in order to being able - * to divide them (ulmo@Q.Net) - * - * Fri Jun 13 10:50:23 CST 1997: Martin Schulze - * Changed definition of LookupSymbol to non-static because it is - * used in klogd.c, too. - * - * Fri Jan 9 23:00:08 CET 1998: Martin Schulze <joey@infodrom.north.de> - * Fixed bug that caused klogd to die if there is no System.map available. - * - * Sun 29 Mar 18:14:07 BST 1998: Mark Simon Phillips <M.S.Phillips@nortel.co.uk> - * Switched to fgets() as gets() is not buffer overrun secure. - * - * Mon Apr 13 18:18:45 CEST 1998: Martin Schulze <joey@infodrom.north.de> - * Modified loop for detecting the correct system map. Now it won't - * stop if a file has been found but doesn't contain the correct map. - * Special thanks go go Mark Simon Phillips for the hint. - * - * Mon Oct 12 00:42:30 CEST 1998: Martin Schulze <joey@infodrom.north.de> - * Modified CheckVersion() - * . Use shift to decode the kernel version - * . Compare integers of kernel version - * . extract major.minor.patch from utsname.release via sscanf() - * The reason lays in possible use of kernel flavours which - * modify utsname.release but no the Version_ symbol. - * - * Sun Feb 21 22:27:49 EST 1999: Keith Owens <kaos@ocs.com.au> - * Fixed bug that caused klogd to die if there is no sym_array available. - * - * Tue Sep 12 23:48:12 CEST 2000: Martin Schulze <joey@infodrom.ffis.de> - * Close symbol file in InitKsyms() when an error occurred. - */ - - -/* Includes. */ -#include "config.h" -#include <stdio.h> -#include <stdlib.h> -#include <sys/utsname.h> -#include <ctype.h> -#include <stdarg.h> -#include <string.h> -#include <syslog.h> -#include "imklog.h" -#include "ksyms.h" -#include "module.h" -#include "debug.h" - - -int num_syms = 0; -static int i_am_paranoid = 0; -static char vstring[12]; -static struct sym_table *sym_array = (struct sym_table *) 0; - -static char *system_maps[] = -{ - "/boot/System.map", - "/System.map", - NULL -}; - - -/* Function prototypes. */ -static char *FindSymbolFile(void); -static int AddSymbol(unsigned long, char*); -static void FreeSymbols(void); -static int CheckVersion(char *); -static int CheckMapVersion(char *); - - -/************************************************************************* - * Function: InitKsyms - * - * Purpose: This function is responsible for initializing and loading - * the data tables used by the kernel address translations. - * - * Arguements: (char *) mapfile - * - * mapfile:-> A pointer to a complete path - * specification of the file containing - * the kernel map to use. - * - * Return: int - * - * A boolean style context is returned. The return value will - * be true if initialization was successful. False if not. - **************************************************************************/ -extern int InitKsyms(char *mapfile) -{ - auto char type, - sym[512]; - - auto int version = 0; - - auto unsigned long int address; - - auto FILE *sym_file; - - BEGINfunc - - /* Check and make sure that we are starting with a clean slate. */ - if ( num_syms > 0 ) - FreeSymbols(); - - - /* Search for and open the file containing the kernel symbols. */ - if ( mapfile != NULL ) { - if ( (sym_file = fopen(mapfile, "r")) == NULL ) - { - imklogLogIntMsg(LOG_WARNING, "Cannot open map file: %s.", mapfile); - return(0); - } - } else { - if ( (mapfile = FindSymbolFile()) == NULL ) { - imklogLogIntMsg(LOG_WARNING, "Cannot find map file."); - dbgprintf("Cannot find map file.\n"); - return(0); - } - - if ( (sym_file = fopen(mapfile, "r")) == NULL ) { - imklogLogIntMsg(LOG_WARNING, "Cannot open map file."); - dbgprintf("Cannot open map file.\n"); - return(0); - } - } - - - /* Read the kernel symbol table file and add entries for each - * line. I suspect that the use of fscanf is not really in vogue - * but it was quick and dirty and IMHO suitable for fixed format - * data such as this. If anybody doesn't agree with this please - * e-mail me a diff containing a parser with suitable political - * correctness -- GW. - */ - while ( !feof(sym_file) ) { - if ( fscanf(sym_file, "%lx %c %s\n", &address, &type, sym) != 3 ) { - imklogLogIntMsg(LOG_ERR, "Error in symbol table input (#1)."); - fclose(sym_file); - return(0); - } - if(dbgPrintSymbols) - dbgprintf("Address: %lx, Type: %c, Symbol: %s\n", address, type, sym); - - if ( AddSymbol(address, sym) == 0 ) { - imklogLogIntMsg(LOG_ERR, "Error adding symbol - %s.", sym); - fclose(sym_file); - return(0); - } - - if ( version == 0 ) - version = CheckVersion(sym); - } - - - imklogLogIntMsg(LOG_INFO, "Loaded %d symbols from %s.", num_syms, mapfile); - switch(version) { - case -1: - imklogLogIntMsg(LOG_WARNING, "Symbols do not match kernel version."); - num_syms = 0; - break; - - case 0: - imklogLogIntMsg(LOG_WARNING, "Cannot verify that symbols match kernel version."); - break; - - case 1: - imklogLogIntMsg(LOG_INFO, "Symbols match kernel version %s.", vstring); - break; - } - - fclose(sym_file); - ENDfunc - return(1); -} - - -extern void DeinitKsyms(void) -{ - FreeSymbols(); -} - - -/************************************************************************** - * Function: FindSymbolFile - * - * Purpose: This function is responsible for encapsulating the search - * for a valid symbol file. Encapsulating the search for - * the map file in this function allows an intelligent search - * process to be implemented. - * - * The list of symbol files will be searched until either a - * symbol file is found whose version matches the currently - * executing kernel or the end of the list is encountered. If - * the end of the list is encountered the first available - * symbol file is returned to the caller. - * - * This strategy allows klogd to locate valid symbol files - * for both a production and an experimental kernel. For - * example a map for a production kernel could be installed - * in /boot. If an experimental kernel is loaded the map - * in /boot will be skipped and the map in /usr/src/linux would - * be used if its version number matches the executing kernel. - * - * Arguements: None specified. - * - * Return: char * - * - * If a valid system map cannot be located a null pointer - * is returned to the caller. - * - * If the search is succesful a pointer is returned to the - * caller which points to the name of the file containing - * the symbol table to be used. - **************************************************************************/ -static char *FindSymbolFile(void) -{ - auto char *file = NULL, - **mf = system_maps; - auto struct utsname utsname; - static char mysymfile[100]; - auto FILE *sym_file = NULL; - BEGINfunc - - if(uname(&utsname) < 0) { - imklogLogIntMsg(LOG_ERR, "Cannot get kernel version information."); - return(0); - } - - dbgprintf("Searching for symbol map.\n"); - - for(mf = system_maps; *mf != NULL && file == NULL; ++mf) { - snprintf(mysymfile, sizeof(mysymfile), "%s-%s", *mf, utsname.release); - dbgprintf("Trying %s.\n", mysymfile); - if((sym_file = fopen(mysymfile, "r")) != NULL) { - if(CheckMapVersion(mysymfile) == 1) - file = mysymfile; - fclose(sym_file); - } - if(sym_file == NULL || file == NULL) { - sprintf (mysymfile, "%s", *mf); - dbgprintf("Trying %s.\n", mysymfile); - if((sym_file = fopen(mysymfile, "r")) != NULL ) { - if (CheckMapVersion(mysymfile) == 1) - file = mysymfile; - fclose(sym_file); - } - } - } - - /* At this stage of the game we are at the end of the symbol tables. */ - dbgprintf("End of search list encountered.\n"); - ENDfunc - return(file); -} - - -/************************************************************************** - * Function: CheckVersion - * - * Purpose: This function is responsible for determining whether or - * the system map being loaded matches the version of the - * currently running kernel. - * - * The kernel version is checked by examing a variable which - * is of the form: _Version_66347 (a.out) or Version_66437 (ELF). - * - * The suffix of this variable is the current kernel version - * of the kernel encoded in base 256. For example the - * above variable would be decoded as: - * - * (66347 = 1*65536 + 3*256 + 43 = 1.3.43) - * - * (Insert appropriate deities here) help us if Linus ever - * needs more than 255 patch levels to get a kernel out the - * door... :-) - * - * Arguements: (char *) version - * - * version:-> A pointer to the string which - * is to be decoded as a kernel - * version variable. - * - * Return: int - * - * -1:-> The currently running kernel version does - * not match this version string. - * - * 0:-> The string is not a kernel version variable. - * - * 1:-> The executing kernel is of the same version - * as the version string. - **************************************************************************/ -static int CheckVersion(char *version) -{ - auto int vnum, - major, - minor, - patch; - int kvnum; - auto struct utsname utsname; - - static char *prefix = { "Version_" }; - - - /* Early return if there is no hope. */ - if ( strncmp(version, prefix, strlen(prefix)) == 0 /* ELF */ || - (*version == '_' && - strncmp(++version, prefix, strlen(prefix)) == 0 ) /* a.out */ ) - ; - else - return(0); - - - /* Since the symbol looks like a kernel version we can start - * things out by decoding the version string into its component - * parts. - */ - vnum = atoi(version + strlen(prefix)); - patch = vnum & 0x000000FF; - minor = (vnum >> 8) & 0x000000FF; - major = (vnum >> 16) & 0x000000FF; - dbgprintf("Version string = %s, Major = %d, Minor = %d, Patch = %d.\n", version + - strlen(prefix), major, minor, patch); - sprintf(vstring, "%d.%d.%d", major, minor, patch); - - /* We should now have the version string in the vstring variable in - * the same format that it is stored in by the kernel. We now - * ask the kernel for its version information and compare the two - * values to determine if our system map matches the kernel - * version level. - */ - if ( uname(&utsname) < 0 ) { - imklogLogIntMsg(LOG_ERR, "Cannot get kernel version information."); - return(0); - } - dbgprintf("Comparing kernel %s with symbol table %s.\n", utsname.release, vstring); - - if ( sscanf (utsname.release, "%d.%d.%d", &major, &minor, &patch) < 3 ) { - imklogLogIntMsg(LOG_ERR, "Kernel send bogus release string `%s'.", utsname.release); - return(0); - } - - /* Compute the version code from data sent by the kernel */ - kvnum = (major << 16) | (minor << 8) | patch; - - /* Failure. */ - if ( vnum != kvnum ) - return(-1); - - /* Success. */ - return(1); -} - - -/************************************************************************** - * Function: CheckMapVersion - * - * Purpose: This function is responsible for determining whether or - * the system map being loaded matches the version of the - * currently running kernel. It uses CheckVersion as - * backend. - * - * Arguements: (char *) fname - * - * fname:-> A pointer to the string which - * references the system map file to - * be used. - * - * Return: int - * - * -1:-> The currently running kernel version does - * not match the version in the given file. - * - * 0:-> No system map file or no version information. - * - * 1:-> The executing kernel is of the same version - * as the version of the map file. - **************************************************************************/ -static int CheckMapVersion(char *fname) -{ - int version; - FILE *sym_file; - auto unsigned long int address; - auto char type, - sym[512]; - - if ( (sym_file = fopen(fname, "r")) != NULL ) { - /* - * At this point a map file was successfully opened. We - * now need to search this file and look for version - * information. - */ - imklogLogIntMsg(LOG_INFO, "Inspecting %s", fname); - - version = 0; - while ( !feof(sym_file) && (version == 0) ) { - if ( fscanf(sym_file, "%lx %c %s\n", &address, &type, sym) != 3 ) { - imklogLogIntMsg(LOG_ERR, "Error in symbol table input (#2)."); - fclose(sym_file); - return(0); - } - if(dbgPrintSymbols) - dbgprintf("Address: %lx, Type: %c, Symbol: %s\n", address, type, sym); - version = CheckVersion(sym); - } - fclose(sym_file); - - switch ( version ) { - case -1: - imklogLogIntMsg(LOG_ERR, "Symbol table has incorrect version number.\n"); - break; - case 0: - dbgprintf("No version information found.\n"); - break; - case 1: - dbgprintf("Found table with matching version number.\n"); - break; - } - - return(version); - } - - return(0); -} - - -/************************************************************************** - * Function: AddSymbol - * - * Purpose: This function is responsible for adding a symbol name - * and its address to the symbol table. - * - * Arguements: (unsigned long) address, (char *) symbol - * - * Return: int - * - * A boolean value is assumed. True if the addition is - * successful. False if not. - **************************************************************************/ -static int AddSymbol(unsigned long address, char *symbol) -{ - /* Allocate the the symbol table entry. */ - sym_array = (struct sym_table *) realloc(sym_array, (num_syms+1) * - sizeof(struct sym_table)); - if ( sym_array == (struct sym_table *) 0 ) - return(0); - - /* Then the space for the symbol. */ - sym_array[num_syms].name = (char *) MALLOC(strlen(symbol)*sizeof(char) + 1); - if ( sym_array[num_syms].name == NULL ) - return(0); - - sym_array[num_syms].value = address; - strcpy(sym_array[num_syms].name, symbol); - ++num_syms; - return(1); -} - - -/************************************************************************** - * Function: LookupSymbol - * - * Purpose: Find the symbol which is related to the given kernel - * address. - * - * Arguements: (long int) value, (struct symbol *) sym - * - * value:-> The address to be located. - * - * sym:-> A pointer to a structure which will be - * loaded with the symbol's parameters. - * - * Return: (char *) - * - * If a match cannot be found a diagnostic string is printed. - * If a match is found the pointer to the symbolic name most - * closely matching the address is returned. - **************************************************************************/ -char * LookupSymbol(unsigned long value, struct symbol *sym) -{ - auto int lp; - - auto char *last; - auto char *name; - - struct symbol ksym, msym; - - if (!sym_array) - return(NULL); - - last = sym_array[0].name; - ksym.offset = 0; - ksym.size = 0; - if ( value < sym_array[0].value ) - return(NULL); - - for(lp = 0; lp <= num_syms; ++lp) { - if ( sym_array[lp].value > value ) { - ksym.offset = value - sym_array[lp-1].value; - ksym.size = sym_array[lp].value - \ - sym_array[lp-1].value; - break; - } - last = sym_array[lp].name; - } - - name = LookupModuleSymbol(value, &msym); - - if ( ksym.offset == 0 && msym.offset == 0 ) { - return(NULL); - } - - if ( ksym.offset == 0 || msym.offset < 0 || - (ksym.offset > 0 && ksym.offset < msym.offset) ) { - sym->offset = ksym.offset; - sym->size = ksym.size; - return(last); - } else { - sym->offset = msym.offset; - sym->size = msym.size; - return(name); - } - - - return(NULL); -} - -/************************************************************************** - * Function: FreeSymbols - * - * Purpose: This function is responsible for freeing all memory which - * has been allocated to hold the static symbol table. It - * also initializes the symbol count and in general prepares - * for a re-read of a static symbol table. - * - * Arguements: void - * - * Return: void - **************************************************************************/ -static void FreeSymbols(void) -{ - auto int lp; - - /* Free each piece of memory allocated for symbol names. */ - for(lp= 0; lp < num_syms; ++lp) - free(sym_array[lp].name); - - /* Whack the entire array and initialize everything. */ - free(sym_array); - sym_array = (struct sym_table *) 0; - num_syms = 0; - - return; -} - - -/************************************************************************** - * Function: LogExpanded - * - * Purpose: This function is responsible for logging a kernel message - * line after all potential numeric kernel addresses have - * been resolved symolically. - * - * Arguements: (char *) line, (char *) el - * - * line:-> A pointer to the buffer containing the kernel - * message to be expanded and logged. - * - * el:-> A pointer to the buffer into which the expanded - * kernel line will be written. - * - * Return: void - **************************************************************************/ -extern char *ExpandKadds(char *line, char *el) -{ - auto char *kp, - *sl = line, - *elp = el, - *symbol; - char num[15]; - auto unsigned long int value; - auto struct symbol sym; - - sym.offset = 0; - sym.size = 0; - - /* - * This is as handy a place to put this as anyplace. - * - * Since the insertion of kernel modules can occur in a somewhat - * dynamic fashion we need some mechanism to insure that the - * kernel symbol tables get read just prior to when they are - * needed. - * - * To accomplish this we look for the Oops string and use its - * presence as a signal to load the module symbols. - * - * This is not the best solution of course, especially if the - * kernel is rapidly going out to lunch. What really needs to - * be done is to somehow generate a callback from the - * kernel whenever a module is loaded or unloaded. I am - * open for patches. - */ - if ( i_am_paranoid && - (strstr(line, "Oops:") != NULL) && !InitMsyms() ) - imklogLogIntMsg(LOG_WARNING, "Cannot load kernel module symbols.\n"); - - - /* - * Early return if there do not appear to be any kernel - * messages in this line. - */ - if ( (num_syms == 0) || - (kp = strstr(line, "[<")) == NULL ) { -#ifdef __sparc__ - if (num_syms) { - /* On SPARC, register dumps do not have the [< >] characters in it. - */ - static struct sparc_tests { - char *str; - int len; - } tests[] = { { "PC: ", 4 }, - { " o7: ", 5 }, - { " ret_pc: ", 9 }, - { " i7: ", 5 }, - { "Caller[", 7 } - }; - int i, j, ndigits; - char *kp2; - for (i = 0; i < 5; i++) { - kp = strstr(line, tests[i].str); - if (!kp) continue; - kp2 = kp + tests[i].len; - if (!isxdigit(*kp2)) continue; - for (ndigits = 1; isxdigit(kp2[ndigits]); ndigits++); - if (ndigits != 8 && ndigits != 16) continue; - /* On sparc64, all kernel addresses are in first 4GB */ - if (ndigits == 16) { - if (strncmp (kp2, "00000000", 8)) continue; - kp2 += 8; - } - if (!i) { - char *kp3; - if (ndigits == 16 && kp > line && kp[-1L] != 'T') continue; - kp3 = kp2 + 8; - if (ndigits == 16) { - if (strncmp (kp3, " TNPC: 00000000", 15) || !isxdigit(kp3[15])) - continue; - kp3 += 15; - } else { - if (strncmp (kp3, " NPC: ", 6) || !isxdigit(kp3[6])) - continue; - kp3 += 6; - } - for (j = 0; isxdigit(kp3[j]); j++); - if (j != 8) continue; - strncpy(elp, line, kp2 + 8 - line); - elp += kp2 + 8 - line; - value = strtol(kp2, (char **) 0, 16); - if ( (symbol = LookupSymbol(value, &sym)) ) { - if (sym.size) - elp += sprintf(elp, " (%s+%d/%d)", symbol, sym.offset, sym.size); - else - elp += sprintf(elp, " (%s)", symbol); - } - strncpy(elp, kp2 + 8, kp3 - kp2); - elp += kp3 - kp2; - value = strtol(kp3, (char **) 0, 16); - if ( (symbol = LookupSymbol(value, &sym)) ) { - if (sym.size) - elp += sprintf(elp, " (%s+%d/%d)", symbol, sym.offset, sym.size); - else - elp += sprintf(elp, " (%s)", symbol); - } - strcpy(elp, kp3 + 8); - } else { - strncpy(elp, line, kp2 + 8 - line); - elp += kp2 + 8 - line; - value = strtol(kp2, (char **) 0, 16); - if ( (symbol = LookupSymbol(value, &sym)) ) { - if (sym.size) - elp += sprintf(elp, " (%s+%d/%d)", symbol, sym.offset, sym.size); - else - elp += sprintf(elp, " (%s)", symbol); - } - strcpy(elp, kp2 + 8); - } - return el; - } - } -#endif - strcpy(el, line); - return(el); - } - - /* Loop through and expand all kernel messages. */ - do { - while ( sl < kp+1 ) - *elp++ = *sl++; - - /* Now poised at a kernel delimiter. */ - if ( (kp = strstr(sl, ">]")) == NULL ) { - strcpy(el, sl); - return(el); - } - strncpy(num,sl+1,kp-sl-1); - num[kp-sl-1] = '\0'; - value = strtoul(num, (char **) 0, 16); - if ( (symbol = LookupSymbol(value, &sym)) == NULL ) - symbol = sl; - - strcat(elp, symbol); - elp += strlen(symbol); - dbgprintf("Symbol: %s = %lx = %s, %x/%d\n", sl+1, value, - (sym.size==0) ? symbol+1 : symbol, sym.offset, sym.size); - - value = 2; - if ( sym.size != 0 ) { - --value; - ++kp; - elp += sprintf(elp, "+0x%x/0x%02x", sym.offset, sym.size); - } - strncat(elp, kp, value); - elp += value; - sl = kp + value; - if ( (kp = strstr(sl, "[<")) == NULL ) - strcat(elp, sl); - } - while ( kp != NULL); - - dbgprintf("Expanded line: %s\n", el); - return(el); -} - - -/************************************************************************** - * Function: SetParanoiaLevel - * - * Purpose: This function is an interface function for setting the - * mode of loadable module symbol lookups. Probably overkill - * but it does slay another global variable. - * - * Arguements: (int) level - * - * level:-> The amount of paranoia which is to be - * present when resolving kernel exceptions. - * Return: void - **************************************************************************/ -extern void SetParanoiaLevel(int level) -{ - i_am_paranoid = level; - return; -} - diff --git a/plugins/imklog/ksym_mod.c b/plugins/imklog/ksym_mod.c deleted file mode 100644 index 82978892..00000000 --- a/plugins/imklog/ksym_mod.c +++ /dev/null @@ -1,485 +0,0 @@ -/* ksym_mod.c - functions for building symbol lookup tables for klogd - * Copyright (c) 1995, 1996 Dr. G.W. Wettstein <greg@wind.rmcc.com> - * Copyright (c) 1996 Enjellic Systems Development - * Copyright (c) 1998-2007 Martin Schulze <joey@infodrom.org> - * Copyright (C) 2007-2009 Rainer Gerhards <rgerhards@adiscon.com> - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. -*/ - -/* - * This file implements functions which are useful for building - * a symbol lookup table based on the in kernel symbol table - * maintained by the Linux kernel. - * - * Proper logging of kernel panics generated by loadable modules - * tends to be difficult. Since the modules are loaded dynamically - * their addresses are not known at kernel load time. A general - * protection fault (Oops) cannot be properly deciphered with - * classic methods using the static symbol map produced at link time. - * - * One solution to this problem is to have klogd attempt to translate - * addresses from module when the fault occurs. By referencing the - * the kernel symbol table proper resolution of these symbols is made - * possible. - * - * At least that is the plan. - * - * Wed Aug 21 09:20:09 CDT 1996: Dr. Wettstein - * The situation where no module support has been compiled into a - * kernel is now detected. An informative message is output indicating - * that the kernel has no loadable module support whenever kernel - * module symbols are loaded. - * - * An informative message is printed indicating the number of kernel - * modules and the number of symbols loaded from these modules. - * - * Sun Jun 15 16:23:29 MET DST 1997: Michael Alan Dorman - * Some more glibc patches made by <mdorman@debian.org>. - * - * Sat Jan 10 15:00:18 CET 1998: Martin Schulze <joey@infodrom.north.de> - * Fixed problem with klogd not being able to be built on a kernel - * newer than 2.1.18. It was caused by modified structures - * inside the kernel that were included. I have worked in a - * patch from Alessandro Suardi <asuardi@uninetcom.it>. - * - * Sun Jan 25 20:57:34 CET 1998: Martin Schulze <joey@infodrom.north.de> - * Another patch for Linux/alpha by Christopher C Chimelis - * <chris@classnet.med.miami.edu>. - * - * Thu Mar 19 23:39:29 CET 1998: Manuel Rodrigues <pmanuel@cindy.fe.up.pt> - * Changed lseek() to llseek() in order to support > 2GB address - * space which provided by kernels > 2.1.70. - * - * Mon Apr 13 18:18:45 CEST 1998: Martin Schulze <joey@infodrom.north.de> - * Removed <sys/module.h> as it's no longer part of recent glibc - * versions. Added prototyp for llseek() which has been - * forgotton in <unistd.h> from glibc. Added more log - * information if problems occurred while reading a system map - * file, by submission from Mark Simon Phillips <M.S.Phillips@nortel.co.uk>. - * - * Sun Jan 3 18:38:03 CET 1999: Martin Schulze <joey@infodrom.north.de> - * Corrected return value of AddModule if /dev/kmem can't be - * loaded. This will prevent klogd from segfaulting if /dev/kmem - * is not available. Patch from Topi Miettinen <tom@medialab.sonera.net>. - * - * Tue Sep 12 23:11:13 CEST 2000: Martin Schulze <joey@infodrom.ffis.de> - * Changed llseek() to lseek64() in order to skip a libc warning. - */ - -/* Includes. */ -#include "config.h" -#include <stdio.h> -#include <stdlib.h> -#include <ctype.h> -#include <unistd.h> -#include <signal.h> -#include <string.h> -#include <errno.h> -#include <sys/fcntl.h> -#include <sys/stat.h> -#if !defined(__GLIBC__) -#include <linux/time.h> -#include <linux/module.h> -#else /* __GLIBC__ */ -#include "module.h" -#endif /* __GLIBC__ */ -#include <stdarg.h> -#include <paths.h> -#include <linux/version.h> - -#include "rsyslog.h" -#include "imklog.h" -#include "ksyms.h" -#include "debug.h" - -#define KSYMS "/proc/kallsyms" - -static int num_modules = 0; -struct Module *sym_array_modules = (struct Module *) NULL; - -static int have_modules = 0; - - -/* Function prototypes. */ -static void FreeModules(void); -static int AddSymbol(const char *); -struct Module *AddModule(const char *); -static int symsort(const void *, const void *); - -/* Imported from ksym.c */ -extern int num_syms; - - -/************************************************************************** - * Function: InitMsyms - * - * Purpose: This function is responsible for building a symbol - * table which can be used to resolve addresses for - * loadable modules. - * - * Arguements: Void - * - * Return: A boolean return value is assumed. - * - * A false value indicates that something went wrong. - * - * True if loading is successful. - **************************************************************************/ -extern int InitMsyms(void) -{ - - auto int rtn, - tmp; - FILE *ksyms; - char buf[128]; - char *p; - - /* Initialize the kernel module symbol table. */ - FreeModules(); - - ksyms = fopen(KSYMS, "r"); - - if ( ksyms == NULL ) { - if ( errno == ENOENT ) - imklogLogIntMsg(LOG_INFO, "No module symbols loaded - " - "kernel modules not enabled.\n"); - else - imklogLogIntMsg(LOG_ERR, "Error loading kernel symbols " \ - "- %s\n", strerror(errno)); - return(0); - } - - dbgprintf("Loading kernel module symbols - Source: %s\n", KSYMS); - - while ( fgets(buf, sizeof(buf), ksyms) != NULL ) { - if (num_syms > 0 && index(buf, '[') == NULL) - continue; - - p = index(buf, ' '); - - if ( p == NULL ) - continue; - - if ( buf[strlen(buf)-1] == '\n' ) - buf[strlen(buf)-1] = '\0'; - /* overlong lines will be ignored above */ - - AddSymbol(buf); - } - - if(ksyms != NULL) - fclose(ksyms); - - have_modules = 1; - - /* Sort the symbol tables in each module. */ - for (rtn = tmp = 0; tmp < num_modules; ++tmp) { - rtn += sym_array_modules[tmp].num_syms; - if ( sym_array_modules[tmp].num_syms < 2 ) - continue; - qsort(sym_array_modules[tmp].sym_array, \ - sym_array_modules[tmp].num_syms, \ - sizeof(struct sym_table), symsort); - } - - if ( rtn == 0 ) - imklogLogIntMsg(LOG_INFO, "No module symbols loaded."); - else - imklogLogIntMsg(LOG_INFO, "Loaded %d %s from %d module%s", rtn, \ - (rtn == 1) ? "symbol" : "symbols", \ - num_modules, (num_modules == 1) ? "." : "s."); - - return(1); -} - - -static int symsort(const void *p1, const void *p2) -{ - auto const struct sym_table *sym1 = p1, - *sym2 = p2; - - if ( sym1->value < sym2->value ) - return(-1); - if ( sym1->value == sym2->value ) - return(0); - return(1); -} - - -extern void DeinitMsyms(void) -{ - FreeModules(); -} - - -/************************************************************************** - * Function: FreeModules - * - * Purpose: This function is used to free all memory which has been - * allocated for the modules and their symbols. - * - * Arguements: None specified. - * - * Return: void - **************************************************************************/ -static void FreeModules() -{ - auto int nmods, - nsyms; - auto struct Module *mp; - - /* Check to see if the module symbol tables need to be cleared. */ - have_modules = 0; - if ( num_modules == 0 ) - return; - - if ( sym_array_modules == NULL ) - return; - - for (nmods = 0; nmods < num_modules; ++nmods) { - mp = &sym_array_modules[nmods]; - if ( mp->num_syms == 0 ) - continue; - - for (nsyms= 0; nsyms < mp->num_syms; ++nsyms) - free(mp->sym_array[nsyms].name); - free(mp->sym_array); - if ( mp->name != NULL ) - free(mp->name); - } - - free(sym_array_modules); - sym_array_modules = (struct Module *) NULL; - num_modules = 0; - return; -} - - -/************************************************************************** - * Function: AddModule - * - * Purpose: This function is responsible for adding a module to - * the list of currently loaded modules. - * - * Arguments: (const char *) module - * - * module:-> The name of the module. - * - * Return: struct Module * - **************************************************************************/ - -struct Module *AddModule(module) - const char *module; -{ - struct Module *mp; - - if ( num_modules == 0 ) { - sym_array_modules = (struct Module *)MALLOC(sizeof(struct Module)); - - if ( sym_array_modules == NULL ) - { - imklogLogIntMsg(LOG_WARNING, "Cannot allocate Module array.\n"); - return NULL; - } - mp = sym_array_modules; - } else { - /* Allocate space for the module. */ - mp = (struct Module *) \ - realloc(sym_array_modules, \ - (num_modules+1) * sizeof(struct Module)); - - if ( mp == NULL ) - { - imklogLogIntMsg(LOG_WARNING, "Cannot allocate Module array.\n"); - return NULL; - } - - sym_array_modules = mp; - mp = &sym_array_modules[num_modules]; - } - - num_modules++; - mp->sym_array = NULL; - mp->num_syms = 0; - - if ( module != NULL ) - mp->name = strdup(module); - else - mp->name = NULL; - - return mp; -} - - -/************************************************************************** - * Function: AddSymbol - * - * Purpose: This function is responsible for adding a symbol name - * and its address to the symbol table. - * - * Arguements: const char * - * - * Return: int - * - * A boolean value is assumed. True if the addition is - * successful. False if not. - **************************************************************************/ -static int AddSymbol(line) - const char *line; -{ - char *module; - unsigned long address; - char *p; - static char *lastmodule = NULL; - struct Module *mp; - - module = index(line, '['); - - if ( module != NULL ) { - p = index(module, ']'); - if ( p != NULL ) - *p = '\0'; - p = module++; - while ( isspace(*(--p)) ) - /*SKIP*/; - *(++p) = '\0'; - } - - p = index(line, ' '); - - if ( p == NULL ) - return(0); - - *p = '\0'; - - address = strtoul(line, (char **) 0, 16); - - p += 3; - - if ( num_modules == 0 || - ( lastmodule == NULL && module != NULL ) || - ( module == NULL && lastmodule != NULL) || - ( module != NULL && strcmp(module, lastmodule))) { - mp = AddModule(module); - - if ( mp == NULL ) - return(0); - } else - mp = &sym_array_modules[num_modules-1]; - - lastmodule = mp->name; - - /* Allocate space for the symbol table entry. */ - mp->sym_array = (struct sym_table *) realloc(mp->sym_array, \ - (mp->num_syms+1) * sizeof(struct sym_table)); - - if ( mp->sym_array == (struct sym_table *) NULL ) - return(0); - - mp->sym_array[mp->num_syms].name = strdup(p); - if ( mp->sym_array[mp->num_syms].name == (char *) NULL ) - return(0); - - /* Stuff interesting information into the module. */ - mp->sym_array[mp->num_syms].value = address; - ++mp->num_syms; - - return(1); -} - - - -/************************************************************************** - * Function: LookupModuleSymbol - * - * Purpose: Find the symbol which is related to the given address from - * a kernel module. - * - * Arguements: (long int) value, (struct symbol *) sym - * - * value:-> The address to be located. - * - * sym:-> A pointer to a structure which will be - * loaded with the symbol's parameters. - * - * Return: (char *) - * - * If a match cannot be found a diagnostic string is printed. - * If a match is found the pointer to the symbolic name most - * closely matching the address is returned. - * - * TODO: We are using int values for the offset, but longs for the value - * values. This may create some trouble in the future (on 64 Bit OS?). - * Anyhow, I have not changed this, because we do not seem to have any - * issue and my understanding of this code is limited (and I don't see - * need to invest more time to dig much deeper). - * rgerhards, 2009-04-17 - **************************************************************************/ -extern char * LookupModuleSymbol(value, sym) - unsigned long value; - struct symbol *sym; -{ - int nmod, nsym; - struct sym_table *last; - struct Module *mp; - static char ret[100]; - - sym->size = 0; - sym->offset = 0; - if ( num_modules == 0 ) - return((char *) 0); - - for (nmod = 0; nmod < num_modules; ++nmod) { - mp = &sym_array_modules[nmod]; - - /* Run through the list of symbols in this module and - * see if the address can be resolved. - */ - for(nsym = 1, last = &mp->sym_array[0]; - nsym < mp->num_syms; - ++nsym) { - if ( mp->sym_array[nsym].value > value ) - { - if ( sym->size == 0 || - (int) (value - last->value) < sym->offset || - ( (sym->offset == (int) (value - last->value)) && - (int) (mp->sym_array[nsym].value-last->value) < sym->size ) ) - { - sym->offset = value - last->value; - sym->size = mp->sym_array[nsym].value - last->value; - ret[sizeof(ret)-1] = '\0'; - if ( mp->name == NULL ) - snprintf(ret, sizeof(ret)-1, - "%s", last->name); - else - snprintf(ret, sizeof(ret)-1, - "%s:%s", mp->name, last->name); - } - break; - } - last = &mp->sym_array[nsym]; - } - } - - if ( sym->size > 0 ) - return(ret); - - /* It has been a hopeless exercise. */ - return(NULL); -} diff --git a/plugins/imklog/ksyms.h b/plugins/imklog/ksyms.h deleted file mode 100644 index a168947b..00000000 --- a/plugins/imklog/ksyms.h +++ /dev/null @@ -1,37 +0,0 @@ -/* ksym.h - Definitions for symbol table utilities. - * Copyright (c) 1995, 1996 Dr. G.W. Wettstein <greg@wind.rmcc.com> - * Copyright (c) 1996 Enjellic Systems Development - * Copyright (c) 2004-7 Martin Schulze <joey@infodrom.org> - * Copyright (c) 2007-2009 Rainer Gerhards <rgerhards@adiscon.com> - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ - -/* Variables, structures and type definitions static to this module. */ - -struct symbol -{ - uchar *name; - int size; - int offset; -}; - - -/* Function prototypes. */ -extern char * LookupSymbol(unsigned long, struct symbol *); -extern char * LookupModuleSymbol(unsigned long int, struct symbol *); diff --git a/plugins/imklog/linux.c b/plugins/imklog/linux.c deleted file mode 100644 index 727708a5..00000000 --- a/plugins/imklog/linux.c +++ /dev/null @@ -1,542 +0,0 @@ -/* klog for linux, based on the FreeBSD syslogd implementation. - * - * This contains OS-specific functionality to read the BSD - * kernel log. For a general overview, see head comment in - * imklog.c. - * - * This file heavily borrows from the klogd daemon provided by - * the sysklogd project. Many thanks for this piece of software. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. -*/ -#include "config.h" -#include "rsyslog.h" -#include <stdlib.h> -#include <stdio.h> -#include <assert.h> -#include <signal.h> -#include <string.h> -#include <pthread.h> -#include "cfsysline.h" -#include "template.h" -#include "msg.h" -#include "module-template.h" -#include "imklog.h" -#include "unicode-helper.h" - - -/* Includes. */ -#include <unistd.h> -#include <errno.h> -#include <sys/fcntl.h> -#include <sys/stat.h> - -#if HAVE_TIME_H -# include <time.h> -#endif - -#include <stdarg.h> -#include <paths.h> -#include "ksyms.h" - -#define __LIBRARY__ -#include <unistd.h> - - -#if !defined(__GLIBC__) -# define __NR_ksyslog __NR_syslog -_syscall3(int,ksyslog,int, type, char *, buf, int, len); -#else -#include <sys/klog.h> -#define ksyslog klogctl -#endif - - - -#ifndef _PATH_KLOG -#define _PATH_KLOG "/proc/kmsg" -#endif - -#define LOG_BUFFER_SIZE 4096 -#define LOG_LINE_LENGTH 1000 - -static int kmsg; -static char log_buffer[LOG_BUFFER_SIZE]; - -static enum LOGSRC {none, proc, kernel} logsrc; - - -/* Function prototypes. */ -extern int ksyslog(int type, char *buf, int len); - - -static uchar *GetPath(void) -{ - return pszPath ? pszPath : UCHAR_CONSTANT(_PATH_KLOG); -} - -static void CloseLogSrc(void) -{ - /* Turn on logging of messages to console, but only if a log level was speficied */ - if(console_log_level != -1) - ksyslog(7, NULL, 0); - - /* Shutdown the log sources. */ - switch(logsrc) { - case kernel: - ksyslog(0, NULL, 0); - imklogLogIntMsg(LOG_INFO, "Kernel logging (ksyslog) stopped."); - break; - case proc: - close(kmsg); - imklogLogIntMsg(LOG_INFO, "Kernel logging (proc) stopped."); - break; - case none: - break; - } - - return; -} - - -static enum LOGSRC GetKernelLogSrc(void) -{ - auto struct stat sb; - - /* Set level of kernel console messaging.. */ - if ( (console_log_level != -1) && - (ksyslog(8, NULL, console_log_level) < 0) && - (errno == EINVAL) ) - { - /* - * An invalid arguement error probably indicates that - * a pre-0.14 kernel is being run. At this point we - * issue an error message and simply shut-off console - * logging completely. - */ - imklogLogIntMsg(LOG_WARNING, "Cannot set console log level - disabling " - "console output."); - } - - /* - * First do a stat to determine whether or not the proc based - * file system is available to get kernel messages from. - */ - if ( use_syscall || - ((stat((char*)GetPath(), &sb) < 0) && (errno == ENOENT)) ) - { - /* Initialize kernel logging. */ - ksyslog(1, NULL, 0); - imklogLogIntMsg(LOG_INFO, "imklog %s, log source = ksyslog " - "started.", VERSION); - return(kernel); - } - - if ( (kmsg = open((char*)GetPath(), O_RDONLY|O_CLOEXEC)) < 0 ) - { - imklogLogIntMsg(LOG_ERR, "imklog: Cannot open proc file system, %d.\n", errno); - ksyslog(7, NULL, 0); - return(none); - } - - imklogLogIntMsg(LOG_INFO, "imklog %s, log source = %s started.", VERSION, GetPath()); - return(proc); -} - - -/* Copy characters from ptr to line until a char in the delim - * string is encountered or until min( space, len ) chars have - * been copied. - * - * Returns the actual number of chars copied. - */ -static int copyin( uchar *line, int space, - const char *ptr, int len, - const char *delim ) -{ - auto int i; - auto int count; - - count = len < space ? len : space; - - for(i=0; i<count && !strchr(delim, *ptr); i++ ) { - *line++ = *ptr++; - } - - return(i); -} - -/* - * Messages are separated by "\n". Messages longer than - * LOG_LINE_LENGTH are broken up. - * - * Kernel symbols show up in the input buffer as : "[<aaaaaa>]", - * where "aaaaaa" is the address. These are replaced with - * "[symbolname+offset/size]" in the output line - symbolname, - * offset, and size come from the kernel symbol table. - * - * If a kernel symbol happens to fall at the end of a message close - * in length to LOG_LINE_LENGTH, the symbol will not be expanded. - * (This should never happen, since the kernel should never generate - * messages that long. - * - * To preserve the original addresses, lines containing kernel symbols - * are output twice. Once with the symbols converted and again with the - * original text. Just in case somebody wants to run their own Oops - * analysis on the syslog, e.g. ksymoops. - */ -static void LogLine(char *ptr, int len) -{ - enum parse_state_enum { - PARSING_TEXT, - PARSING_SYMSTART, /* at < */ - PARSING_SYMBOL, - PARSING_SYMEND /* at ] */ - }; - - static uchar line_buff[LOG_LINE_LENGTH]; - - static uchar *line =line_buff; - static enum parse_state_enum parse_state = PARSING_TEXT; - static int space = sizeof(line_buff)-1; - - static uchar *sym_start; /* points at the '<' of a symbol */ - - auto int delta = 0; /* number of chars copied */ - auto int symbols_expanded = 0; /* 1 if symbols were expanded */ - auto int skip_symbol_lookup = 0; /* skip symbol lookup on this pass */ - auto char *save_ptr = ptr; /* save start of input line */ - auto int save_len = len; /* save length at start of input line */ - - while( len > 0 ) - { - if( space == 0 ) /* line buffer is full */ - { - /* - ** Line too long. Start a new line. - */ - *line = 0; /* force null terminator */ - - //dbgprintf("Line buffer full:\n"); - //dbgprintf("\tLine: %s\n", line); - - Syslog(LOG_INFO, line_buff); - line = line_buff; - space = sizeof(line_buff)-1; - parse_state = PARSING_TEXT; - symbols_expanded = 0; - skip_symbol_lookup = 0; - save_ptr = ptr; - save_len = len; - } - - switch( parse_state ) - { - case PARSING_TEXT: - delta = copyin(line, space, ptr, len, "\n[" ); - line += delta; - ptr += delta; - space -= delta; - len -= delta; - - if( space == 0 || len == 0 ) - { - break; /* full line_buff or end of input buffer */ - } - - if( *ptr == '\0' ) /* zero byte */ - { - ptr++; /* skip zero byte */ - space -= 1; - len -= 1; - - break; - } - - if( *ptr == '\n' ) /* newline */ - { - ptr++; /* skip newline */ - space -= 1; - len -= 1; - - *line = 0; /* force null terminator */ - Syslog(LOG_INFO, line_buff); - line = line_buff; - space = sizeof(line_buff)-1; - if (symbols_twice) { - if (symbols_expanded) { - /* reprint this line without symbol lookup */ - symbols_expanded = 0; - skip_symbol_lookup = 1; - ptr = save_ptr; - len = save_len; - } - else - { - skip_symbol_lookup = 0; - save_ptr = ptr; - save_len = len; - } - } - break; - } - if( *ptr == '[' ) /* possible kernel symbol */ - { - *line++ = *ptr++; - space -= 1; - len -= 1; - if (!skip_symbol_lookup) - parse_state = PARSING_SYMSTART; /* at < */ - break; - } - /* Now that line_buff is no longer fed to *printf as format - * string, '%'s are no longer "dangerous". - */ - break; - - case PARSING_SYMSTART: - if( *ptr != '<' ) - { - parse_state = PARSING_TEXT; /* not a symbol */ - break; - } - - /* - ** Save this character for now. If this turns out to - ** be a valid symbol, this char will be replaced later. - ** If not, we'll just leave it there. - */ - - sym_start = line; /* this will point at the '<' */ - - *line++ = *ptr++; - space -= 1; - len -= 1; - parse_state = PARSING_SYMBOL; /* symbol... */ - break; - - case PARSING_SYMBOL: - delta = copyin( line, space, ptr, len, ">\n[" ); - line += delta; - ptr += delta; - space -= delta; - len -= delta; - if( space == 0 || len == 0 ) - { - break; /* full line_buff or end of input buffer */ - } - if( *ptr != '>' ) - { - parse_state = PARSING_TEXT; - break; - } - - *line++ = *ptr++; /* copy the '>' */ - space -= 1; - len -= 1; - - parse_state = PARSING_SYMEND; - - break; - - case PARSING_SYMEND: - if( *ptr != ']' ) - { - parse_state = PARSING_TEXT; /* not a symbol */ - break; - } - - /* - ** It's really a symbol! Replace address with the - ** symbol text. - */ - { - auto int sym_space; - - unsigned long value; - auto struct symbol sym; - auto char *symbol; - - *(line-1) = 0; /* null terminate the address string */ - value = strtoul((char*)(sym_start+1), (char **) 0, 16); - *(line-1) = '>'; /* put back delim */ - - if ( !symbol_lookup || (symbol = LookupSymbol(value, &sym)) == (char *)0 ) - { - parse_state = PARSING_TEXT; - break; - } - - /* - ** verify there is room in the line buffer - */ - sym_space = space + ( line - sym_start ); - if( (unsigned) sym_space < strlen(symbol) + 30 ) /*(30 should be overkill)*/ - { - parse_state = PARSING_TEXT; /* not enough space */ - break; - } - - // TODO: sprintf!!!! - delta = sprintf( (char*) sym_start, "%s+%d/%d]", - symbol, sym.offset, sym.size ); - - space = sym_space + delta; - line = sym_start + delta; - symbols_expanded = 1; - } - ptr++; - len--; - parse_state = PARSING_TEXT; - break; - - default: /* Can't get here! */ - parse_state = PARSING_TEXT; - - } - } - - return; -} - - -static void LogKernelLine(void) -{ - auto int rdcnt; - - /* - * Zero-fill the log buffer. This should cure a multitude of - * problems with klogd logging the tail end of the message buffer - * which will contain old messages. Then read the kernel log - * messages into this fresh buffer. - */ - memset(log_buffer, '\0', sizeof(log_buffer)); - if ( (rdcnt = ksyslog(2, log_buffer, sizeof(log_buffer)-1)) < 0 ) - { - if(errno == EINTR) - return; - imklogLogIntMsg(LOG_ERR, "imklog Error return from sys_sycall: %d\n", errno); - } - else - LogLine(log_buffer, rdcnt); - return; -} - - -static void LogProcLine(void) -{ - auto int rdcnt; - - /* - * Zero-fill the log buffer. This should cure a multitude of - * problems with klogd logging the tail end of the message buffer - * which will contain old messages. Then read the kernel messages - * from the message pseudo-file into this fresh buffer. - */ - memset(log_buffer, '\0', sizeof(log_buffer)); - if ( (rdcnt = read(kmsg, log_buffer, sizeof(log_buffer)-1)) < 0 ) { - if ( errno == EINTR ) - return; - imklogLogIntMsg(LOG_ERR, "Cannot read proc file system: %d - %s.", errno, strerror(errno)); - } else { - LogLine(log_buffer, rdcnt); - } - - return; -} - - -/* to be called in the module's WillRun entry point - * rgerhards, 2008-04-09 - */ -rsRetVal klogLogKMsg(void) -{ - DEFiRet; - switch(logsrc) { - case kernel: - LogKernelLine(); - break; - case proc: - LogProcLine(); - break; - case none: - /* TODO: We need to handle this case here somewhat more intelligent - * This is now at least partly done - code should never reach this point - * as willRun() already checked for the "none" status -- rgerhards, 2007-12-17 - */ - pause(); - break; - } - RETiRet; -} - - -/* to be called in the module's WillRun entry point - * rgerhards, 2008-04-09 - */ -rsRetVal klogWillRun(void) -{ - DEFiRet; - /* Initialize this module. If that fails, we tell the engine we don't like to run */ - /* Determine where kernel logging information is to come from. */ - logsrc = GetKernelLogSrc(); - if(logsrc == none) { - iRet = RS_RET_NO_KERNEL_LOGSRC; - } else { - if (symbol_lookup) { - symbol_lookup = (InitKsyms(symfile) == 1); - symbol_lookup |= InitMsyms(); - if (symbol_lookup == 0) { - imklogLogIntMsg(LOG_WARNING, "cannot find any symbols, turning off symbol lookups"); - } - } - } - - RETiRet; -} - - -/* to be called in the module's AfterRun entry point - * rgerhards, 2008-04-09 - */ -rsRetVal klogAfterRun(void) -{ - DEFiRet; - /* cleanup here */ - if(logsrc != none) - CloseLogSrc(); - - DeinitKsyms(); - DeinitMsyms(); - - RETiRet; -} - - -/* provide the (system-specific) default facility for internal messages - * rgerhards, 2008-04-14 - */ -int -klogFacilIntMsg(void) -{ - return LOG_KERN; -} - - -/* vi:set ai: - */ diff --git a/plugins/imklog/module.h b/plugins/imklog/module.h deleted file mode 100644 index 38a26fea..00000000 --- a/plugins/imklog/module.h +++ /dev/null @@ -1,35 +0,0 @@ -/* module.h - Miscellaneous module definitions - * Copyright (c) 1996 Richard Henderson <rth@tamu.edu> - * Copyright (c) 2004-7 Martin Schulze <joey@infodrom.org> - * Copyright (c) 2007-2008 Rainer Gerhards <rgerhards@adiscon.com> - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -struct sym_table -{ - unsigned long value; - char *name; -}; - -struct Module -{ - struct sym_table *sym_array; - int num_syms; - - char *name; -}; diff --git a/plugins/imklog/solaris.c b/plugins/imklog/solaris.c index 8a6d5af1..0a169cdd 100644 --- a/plugins/imklog/solaris.c +++ b/plugins/imklog/solaris.c @@ -80,74 +80,6 @@ klogWillRun(void) } -#if 0 -/* Read /dev/klog while data are available, split into lines. - * Contrary to standard BSD syslogd, we do a blocking read. We can - * afford this as imklog is running on its own threads. So if we have - * a single file, it really doesn't matter if we wait inside a 1-file - * select or the read() directly. - */ -static void -readklog(void) -{ - char *p, *q; - int len, i; - int iMaxLine; - uchar bufRcv[4096+1]; - uchar *pRcv = NULL; /* receive buffer */ - - iMaxLine = klog_getMaxLine(); - - /* we optimize performance: if iMaxLine is below 4K (which it is in almost all - * cases, we use a fixed buffer on the stack. Only if it is higher, heap memory - * is used. We could use alloca() to achive a similar aspect, but there are so - * many issues with alloca() that I do not want to take that route. - * rgerhards, 2008-09-02 - */ - if((size_t) iMaxLine < sizeof(bufRcv) - 1) { - pRcv = bufRcv; - } else { - if((pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))) == NULL) - iMaxLine = sizeof(bufRcv) - 1; /* better this than noting */ - } - - len = 0; - for (;;) { - dbgprintf("----------imklog(BSD) waiting for kernel log line\n"); - i = read(fklog, pRcv + len, iMaxLine - len); - if (i > 0) { - pRcv[i + len] = '\0'; - } else { - if (i < 0 && errno != EINTR && errno != EAGAIN) { - imklogLogIntMsg(LOG_ERR, - "imklog error %d reading kernel log - shutting down imklog", - errno); - fklog = -1; - } - break; - } - - for(p = pRcv; (q = strchr(p, '\n')) != NULL; p = q + 1) { - *q = '\0'; - Syslog(LOG_INFO, (uchar*) p); - } - len = strlen(p); - if (len >= iMaxLine - 1) { - Syslog(LOG_INFO, (uchar*)p); - len = 0; - } - if (len > 0) - memmove(pRcv, p, len + 1); - } - if (len > 0) - Syslog(LOG_INFO, pRcv); - - if(pRcv != NULL && (size_t) iMaxLine >= sizeof(bufRcv) - 1) - free(pRcv); -} -#endif - - /* to be called in the module's AfterRun entry point * rgerhards, 2008-04-09 */ diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 103da210..a735b0b2 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -49,6 +49,7 @@ #include <sys/types.h> #include <sys/socket.h> #include <sys/epoll.h> +#include <netinet/tcp.h> #if HAVE_FCNTL_H #include <fcntl.h> #endif @@ -88,6 +89,10 @@ DEFobjCurrIf(statsobj) /* config settings */ typedef struct configSettings_s { + int bKeepAlive; /* support keep-alive packets */ + int iKeepAliveIntvl; + int iKeepAliveProbes; + int iKeepAliveTime; int bEmitMsgOnClose; /* emit an informational message on close by remote peer */ int iAddtlFrameDelim; /* addtl frame delimiter, e.g. for netscreen, default none */ uchar *pszInputName; /* value for inputname property, NULL is OK and handled by core engine */ @@ -111,13 +116,17 @@ struct ptcpsrv_s { ptcpsrv_t *pNext; /* linked list maintenance */ uchar *port; /* Port to listen to */ uchar *lstnIP; /* which IP we should listen on? */ - int bEmitMsgOnClose; int iAddtlFrameDelim; + int iKeepAliveIntvl; + int iKeepAliveProbes; + int iKeepAliveTime; uchar *pszInputName; prop_t *pInputName; /* InputName in (fast to process) property format */ ruleset_t *pRuleset; ptcplstn_t *pLstn; /* root of our listeners */ ptcpsess_t *pSess; /* root of our sessions */ + sbool bKeepAlive; /* support keep-alive packets */ + sbool bEmitMsgOnClose; }; /* the ptcp session object. Describes a single active session. @@ -434,12 +443,80 @@ finalize_it: } +/* Enable KEEPALIVE handling on the socket. */ +static inline rsRetVal +EnableKeepAlive(ptcplstn_t *pLstn, int sock) +{ + int ret; + int optval; + socklen_t optlen; + DEFiRet; + + optval = 1; + optlen = sizeof(optval); + ret = setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen); + if(ret < 0) { + dbgprintf("EnableKeepAlive socket call returns error %d\n", ret); + ABORT_FINALIZE(RS_RET_ERR); + } + +# if defined(TCP_KEEPCNT) + if(pLstn->pSrv->iKeepAliveProbes > 0) { + optval = pLstn->pSrv->iKeepAliveProbes; + optlen = sizeof(optval); + ret = setsockopt(sock, SOL_TCP, TCP_KEEPCNT, &optval, optlen); + } else { + ret = 0; + } +# else + ret = -1; +# endif + if(ret < 0) { + errmsg.LogError(ret, NO_ERRCODE, "imptcp cannot set keepalive probes - ignored"); + } + +# if defined(TCP_KEEPCNT) + if(pLstn->pSrv->iKeepAliveTime > 0) { + optval = pLstn->pSrv->iKeepAliveTime; + optlen = sizeof(optval); + ret = setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, &optval, optlen); + } else { + ret = 0; + } +# else + ret = -1; +# endif + if(ret < 0) { + errmsg.LogError(ret, NO_ERRCODE, "imptcp cannot set keepalive time - ignored"); + } + +# if defined(TCP_KEEPCNT) + if(pLstn->pSrv->iKeepAliveIntvl > 0) { + optval = pLstn->pSrv->iKeepAliveIntvl; + optlen = sizeof(optval); + ret = setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, &optval, optlen); + } else { + ret = 0; + } +# else + ret = -1; +# endif + if(ret < 0) { + errmsg.LogError(errno, NO_ERRCODE, "imptcp cannot set keepalive intvl - ignored"); + } + + dbgprintf("KEEPALIVE enabled for socket %d\n", sock); + +finalize_it: + RETiRet; +} + /* accept an incoming connection request * rgerhards, 2008-04-22 */ static rsRetVal -AcceptConnReq(int sock, int *newSock, prop_t **peerName, prop_t **peerIP) +AcceptConnReq(ptcplstn_t *pLstn, int *newSock, prop_t **peerName, prop_t **peerIP) { int sockflags; struct sockaddr_storage addr; @@ -448,13 +525,17 @@ AcceptConnReq(int sock, int *newSock, prop_t **peerName, prop_t **peerIP) DEFiRet; - iNewSock = accept(sock, (struct sockaddr*) &addr, &addrlen); + iNewSock = accept(pLstn->sock, (struct sockaddr*) &addr, &addrlen); if(iNewSock < 0) { if(errno == EAGAIN || errno == EWOULDBLOCK) ABORT_FINALIZE(RS_RET_NO_MORE_DATA); ABORT_FINALIZE(RS_RET_ACCEPT_ERR); } + if(pLstn->pSrv->bKeepAlive) + EnableKeepAlive(pLstn, iNewSock);/* we ignore errors, best to do! */ + + CHKiRet(getPeerNames(peerName, peerIP, (struct sockaddr*) &addr)); /* set the new socket to non-blocking IO */ @@ -904,6 +985,10 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKmalloc(pSrv = malloc(sizeof(ptcpsrv_t))); pSrv->pSess = NULL; pSrv->pLstn = NULL; + pSrv->bKeepAlive = cs.bKeepAlive; + pSrv->iKeepAliveIntvl = cs.iKeepAliveTime; + pSrv->iKeepAliveProbes = cs.iKeepAliveProbes; + pSrv->iKeepAliveTime = cs.iKeepAliveTime; pSrv->bEmitMsgOnClose = cs.bEmitMsgOnClose; pSrv->port = pNewVal; pSrv->iAddtlFrameDelim = cs.iAddtlFrameDelim; @@ -967,7 +1052,7 @@ lstnActivity(ptcplstn_t *pLstn) DBGPRINTF("imptcp: new connection on listen socket %d\n", pLstn->sock); while(1) { - localRet = AcceptConnReq(pLstn->sock, &newSock, &peerName, &peerIP); + localRet = AcceptConnReq(pLstn, &newSock, &peerName, &peerIP); if(localRet == RS_RET_NO_MORE_DATA) break; CHKiRet(localRet); @@ -1169,6 +1254,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { cs.bEmitMsgOnClose = 0; + cs.bKeepAlive = 0; + cs.iKeepAliveProbes = 0; + cs.iKeepAliveTime = 0; + cs.iKeepAliveIntvl = 0; cs.iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; free(cs.pszInputName); cs.pszInputName = NULL; @@ -1202,6 +1291,14 @@ CODEmodInit_QueryRegCFSLineHdlr /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputptcpserverrun"), 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputptcpserverkeepalive"), 0, eCmdHdlrBinary, + NULL, &cs.bKeepAlive, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputptcpserverkeepalive_probes"), 0, eCmdHdlrInt, + NULL, &cs.iKeepAliveProbes, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputptcpserverkeepalive_time"), 0, eCmdHdlrInt, + NULL, &cs.iKeepAliveTime, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputptcpserverkeepalive_intvl"), 0, eCmdHdlrInt, + NULL, &cs.iKeepAliveIntvl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputptcpservernotifyonconnectionclose"), 0, eCmdHdlrBinary, NULL, &cs.bEmitMsgOnClose, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputptcpserveraddtlframedelimiter"), 0, eCmdHdlrInt, diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 6ab39477..e9961af7 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -82,12 +82,14 @@ static permittedPeers_t *pPermPeersRoot = NULL; /* config settings */ +static int bKeepAlive = 0; /* support keep-alive packets */ static int iTCPSessMax = 200; /* max number of sessions */ static int iTCPLstnMax = 20; /* max number of sessions */ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ static int bEmitMsgOnClose = 0; /* emit an informational message on close by remote peer */ static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ static int bDisableLFDelim = 0; /* disbale standard LF delimiter */ +static int bUseFlowControl = 1; /* use flow control, what means indicate ourselfs a "light delayable" */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ @@ -191,6 +193,7 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa if(pOurTcpsrv == NULL) { CHKiRet(tcpsrv.Construct(&pOurTcpsrv)); + CHKiRet(tcpsrv.SetKeepAlive(pOurTcpsrv, bKeepAlive)); CHKiRet(tcpsrv.SetSessMax(pOurTcpsrv, iTCPSessMax)); CHKiRet(tcpsrv.SetLstnMax(pOurTcpsrv, iTCPLstnMax)); CHKiRet(tcpsrv.SetCBIsPermittedHost(pOurTcpsrv, isPermittedHost)); @@ -199,6 +202,7 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); + CHKiRet(tcpsrv.SetUseFlowControl(pOurTcpsrv, bUseFlowControl)); CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); CHKiRet(tcpsrv.SetbDisableLFDelim(pOurTcpsrv, bDisableLFDelim)); CHKiRet(tcpsrv.SetNotificationOnRemoteClose(pOurTcpsrv, bEmitMsgOnClose)); @@ -287,8 +291,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { iTCPSessMax = 200; + bKeepAlive = 0; iTCPLstnMax = 20; iStrmDrvrMode = 0; + bUseFlowControl = 0; bEmitMsgOnClose = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; bDisableLFDelim = 0; @@ -324,6 +330,8 @@ CODEmodInit_QueryRegCFSLineHdlr /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverrun"), 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverkeepalive"), 0, eCmdHdlrBinary, + NULL, &bKeepAlive, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpmaxsessions"), 0, eCmdHdlrInt, NULL, &iTCPSessMax, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpmaxlisteners"), 0, eCmdHdlrInt, @@ -344,6 +352,8 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverbindruleset"), 0, eCmdHdlrGetWord, setRuleset, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpflowcontrol"), 0, + eCmdHdlrBinary, NULL, &bUseFlowControl, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index feddb20c..403173e1 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -34,6 +34,7 @@ #include <string.h> #include <errno.h> #include <unistd.h> +#include <fcntl.h> #include <sys/stat.h> #include <sys/un.h> #include <sys/socket.h> @@ -135,7 +136,9 @@ typedef struct lstn_s { sbool bParseHost; /* should parser parse host name? read-only after startup */ sbool bCreatePath; /* auto-creation of socket directory? */ sbool bUseCreds; /* pull original creator credentials from socket */ + sbool bAnnotate; /* annotate events with trusted properties */ sbool bWritePid; /* write original PID into tag */ + sbool bUseSysTimeStamp; /* use timestamp from system (instead of from message) */ } lstn_t; static lstn_t listeners[MAXFUNIX]; @@ -156,9 +159,13 @@ static int bUseFlowCtl = 0; /* use flow control or not (if yes, only LIGHT is u static int bIgnoreTimestamp = 1; /* ignore timestamps present in the incoming message? */ static int bWritePid = 0; /* use credentials from recvmsg() and fixup PID in TAG */ static int bWritePidSysSock = 0; /* use credentials from recvmsg() and fixup PID in TAG */ +static int bUseSysTimeStamp = 1; /* use timestamp from system (rather than from message) */ +static int bUseSysTimeStampSysSock = 1; /* same, for system log socket */ +static int bAnnotate = 0; /* annotate trusted properties */ +static int bAnnotateSysSock = 0; /* same, for system log socket */ #define DFLT_bCreatePath 0 static int bCreatePath = DFLT_bCreatePath; /* auto-create socket path? */ -#define DFLT_ratelimitInterval 5 +#define DFLT_ratelimitInterval 0 static int ratelimitInterval = DFLT_ratelimitInterval; /* interval in seconds, 0 = off */ static int ratelimitIntervalSysSock = DFLT_ratelimitInterval; #define DFLT_ratelimitBurst 200 @@ -300,8 +307,10 @@ addLstnSocketName(void __attribute__((unused)) *pVal, uchar *pNewVal) listeners[nfd].flags = bIgnoreTimestamp ? IGNDATE : NOFLAG; listeners[nfd].bCreatePath = bCreatePath; listeners[nfd].sockName = pNewVal; - listeners[nfd].bUseCreds = (bWritePid || ratelimitInterval) ? 1 : 0; + listeners[nfd].bUseCreds = (bWritePid || ratelimitInterval || bAnnotate) ? 1 : 0; + listeners[nfd].bAnnotate = bAnnotate; listeners[nfd].bWritePid = bWritePid; + listeners[nfd].bUseSysTimeStamp = bUseSysTimeStamp; nfd++; } else { errmsg.LogError(0, NO_ERRCODE, "Out of unix socket name descriptors, ignoring %s\n", @@ -415,9 +424,14 @@ openLogSocket(lstn_t *pLstn) errmsg.LogError(errno, NO_ERRCODE, "set SCM_CREDENTIALS failed on '%s'", pLstn->sockName); pLstn->bUseCreds = 0; } +// TODO: move to its own #if + if(setsockopt(pLstn->fd, SOL_SOCKET, SO_TIMESTAMP, &one, sizeof(one)) != 0) { + errmsg.LogError(errno, NO_ERRCODE, "set SO_TIMESTAMP failed on '%s'", pLstn->sockName); + } } # else /* HAVE_SCM_CREDENTIALS */ pLstn->bUseCreds = 0; + pLstn->bAnnotate = 0; # endif /* HAVE_SCM_CREDENTIALS */ finalize_it: @@ -500,12 +514,109 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) } +/* Get an "trusted property" from the system. Returns an empty string if the + * property can not be obtained. Inspired by similiar functionality inside + * journald. Currently works with Linux /proc filesystem, only. + */ +static rsRetVal +getTrustedProp(struct ucred *cred, char *propName, uchar *buf, size_t lenBuf, int *lenProp) +{ + int fd; + int i; + int lenRead; + char namebuf[1024]; + DEFiRet; + + if(snprintf(namebuf, sizeof(namebuf), "/proc/%lu/%s", (long unsigned) cred->pid, + propName) >= (int) sizeof(namebuf)) { + ABORT_FINALIZE(RS_RET_ERR); + } + + if((fd = open(namebuf, O_RDONLY)) == -1) { + DBGPRINTF("error reading '%s'\n", namebuf); + *lenProp = 0; + FINALIZE; + } + if((lenRead = read(fd, buf, lenBuf - 1)) == -1) { + DBGPRINTF("error reading file data for '%s'\n", namebuf); + *lenProp = 0; + close(fd); + FINALIZE; + } + + /* we strip after the first \n */ + for(i = 0 ; i < lenRead ; ++i) { + if(buf[i] == '\n') + break; + else if(iscntrl(buf[i])) + buf[i] = ' '; + } + buf[i] = '\0'; + *lenProp = i; + + close(fd); + +finalize_it: + RETiRet; +} + + +/* read the exe trusted property path (so far, /proc fs only) + */ +static rsRetVal +getTrustedExe(struct ucred *cred, uchar *buf, size_t lenBuf, int* lenProp) +{ + int lenRead; + char namebuf[1024]; + DEFiRet; + + if(snprintf(namebuf, sizeof(namebuf), "/proc/%lu/exe", (long unsigned) cred->pid) + >= (int) sizeof(namebuf)) { + ABORT_FINALIZE(RS_RET_ERR); + } + + if((lenRead = readlink(namebuf, (char*)buf, lenBuf - 1)) == -1) { + DBGPRINTF("error reading link '%s'\n", namebuf); + *lenProp = 0; + FINALIZE; + } + + buf[lenRead] = '\0'; + *lenProp = lenRead; + +finalize_it: + RETiRet; +} + + +/* copy a trusted property in escaped mode. That is, the property can contain + * any character and so it must be properly quoted AND escaped. + * It is assumed the output buffer is large enough. Returns the number of + * characters added. + */ +static inline int +copyescaped(uchar *dstbuf, uchar *inbuf, int inlen) +{ + int iDst, iSrc; + + *dstbuf = '"'; + for(iDst=1, iSrc=0 ; iSrc < inlen ; ++iDst, ++iSrc) { + if(inbuf[iSrc] == '"' || inbuf[iSrc] == '\\') { + dstbuf[iDst++] = '\\'; + } + dstbuf[iDst] = inbuf[iSrc]; + } + dstbuf[iDst++] = '"'; + return iDst; +} + + /* submit received message to the queue engine * We now parse the message according to expected format so that we * can also mangle it if necessary. */ static inline rsRetVal -SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) +SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct timeval *ts) { msg_t *pMsg; int lenMsg; @@ -519,6 +630,12 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) struct syslogTime st; time_t tt; rs_ratelimit_state_t *ratelimiter = NULL; + int lenProp; + uchar propBuf[1024]; + uchar msgbuf[8192]; + uchar *pmsgbuf; + int toffs; /* offset for trusted properties */ + struct syslogTime dummyTS; DEFiRet; /* TODO: handle format errors?? */ @@ -544,12 +661,58 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ } - datetime.getCurrTime(&st, &tt); + if(ts == NULL) { + datetime.getCurrTime(&st, &tt); + } else { + datetime.timeval2syslogTime(ts, &st); + tt = ts->tv_sec; + } + if(ratelimiter != NULL && !withinRatelimit(ratelimiter, tt, cred->pid)) { STATSCOUNTER_INC(ctrLostRatelimit, mutCtrLostRatelimit); FINALIZE; } + /* created trusted properties */ + if(cred != NULL && pLstn->bAnnotate) { + if((unsigned) (lenRcv + 4096) < sizeof(msgbuf)) { + pmsgbuf = msgbuf; + } else { + CHKmalloc(pmsgbuf = malloc(lenRcv+4096)); + } + memcpy(pmsgbuf, pRcv, lenRcv); + memcpy(pmsgbuf+lenRcv, " @[", 3); + toffs = lenRcv + 3; /* next free location */ + lenProp = snprintf((char*)propBuf, sizeof(propBuf), "_PID=%lu _UID=%lu _GID=%lu", + (long unsigned) cred->pid, (long unsigned) cred->uid, + (long unsigned) cred->gid); + memcpy(pmsgbuf+toffs, propBuf, lenProp); + toffs = toffs + lenProp; + getTrustedProp(cred, "comm", propBuf, sizeof(propBuf), &lenProp); + if(lenProp) { + memcpy(pmsgbuf+toffs, " _COMM=", 7); + memcpy(pmsgbuf+toffs+7, propBuf, lenProp); + toffs = toffs + 7 + lenProp; + } + getTrustedExe(cred, propBuf, sizeof(propBuf), &lenProp); + if(lenProp) { + memcpy(pmsgbuf+toffs, " _EXE=", 6); + memcpy(pmsgbuf+toffs+6, propBuf, lenProp); + toffs = toffs + 6 + lenProp; + } + getTrustedProp(cred, "cmdline", propBuf, sizeof(propBuf), &lenProp); + if(lenProp) { + memcpy(pmsgbuf+toffs, " _CMDLINE=", 10); + toffs = toffs + 10 + + copyescaped(pmsgbuf+toffs+10, propBuf, lenProp); + } + /* finalize string */ + pmsgbuf[toffs] = ']'; + pmsgbuf[toffs+1] = '\0'; + pRcv = pmsgbuf; + lenRcv = toffs + 1; + } + /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &st, tt)); MsgSetRawMsg(pMsg, (char*)pRcv, lenRcv); @@ -564,15 +727,27 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred) parse++; lenMsg--; /* '>' */ - if((pLstn->flags & IGNDATE)) { - /* in this case, we still need to find out if we have a valid - * datestamp or not .. and advance the parse pointer accordingly. - */ - struct syslogTime dummy; - datetime.ParseTIMESTAMP3164(&dummy, &parse, &lenMsg); - } else { - if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { - DBGPRINTF("we have a problem, invalid timestamp in msg!\n"); + if(ts == NULL) { + if((pLstn->flags & IGNDATE)) { + /* in this case, we still need to find out if we have a valid + * datestamp or not .. and advance the parse pointer accordingly. + */ + datetime.ParseTIMESTAMP3164(&dummyTS, &parse, &lenMsg); + } else { + if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &parse, &lenMsg) != RS_RET_OK) { + DBGPRINTF("we have a problem, invalid timestamp in msg!\n"); + } + } + } else { /* if we pulled the time from the system, we need to update the message text */ + uchar *tmpParse = parse; /* just to check correctness of TS */ + if(datetime.ParseTIMESTAMP3164(&dummyTS, &tmpParse, &lenMsg) == RS_RET_OK) { + /* We modify the message only if it contained a valid timestamp, + * otherwise we do not touch it at all. */ + datetime.formatTimestamp3164(&st, (char*)parse, 0); + parse[15] = ' '; /* re-write \0 from fromatTimestamp3164 by SP */ + /* update "counters" to reflect processed timestamp */ + parse += 16; + lenMsg -= 16; } } @@ -624,6 +799,7 @@ static rsRetVal readSocket(lstn_t *pLstn) struct cmsghdr *cm; # endif struct ucred *cred; + struct timeval *ts; uchar bufRcv[4096+1]; char aux[128]; uchar *pRcv = NULL; /* receive buffer */ @@ -662,21 +838,28 @@ static rsRetVal readSocket(lstn_t *pLstn) dbgprintf("Message from UNIX socket: #%d\n", pLstn->fd); if(iRcvd > 0) { cred = NULL; -# if HAVE_SCM_CREDENTIALS - if(pLstn->bUseCreds) { - dbgprintf("XXX: pre CM loop, length of control message %d\n", (int) msgh.msg_controllen); - for (cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { - dbgprintf("XXX: in CM loop, %d, %d\n", cm->cmsg_level, cm->cmsg_type); - if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { + ts = NULL; + if(pLstn->bUseCreds || pLstn->bUseSysTimeStamp) { + for(cm = CMSG_FIRSTHDR(&msgh); cm; cm = CMSG_NXTHDR(&msgh, cm)) { +# if HAVE_SCM_CREDENTIALS + if( pLstn->bUseCreds + && cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_CREDENTIALS) { cred = (struct ucred*) CMSG_DATA(cm); - dbgprintf("XXX: got credentials pid %d\n", (int) cred->pid); break; } +# endif /* HAVE_SCM_CREDENTIALS */ +# if HAVE_SO_TIMESTAMP + if( pLstn->bUseSysTimeStamp + && cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SO_TIMESTAMP) { + ts = (struct timeval *)CMSG_DATA(cm); + dbgprintf("XXX: got timestamp %ld.%ld\n", + (long) ts->tv_sec, (long) ts->tv_usec); + break; + } +# endif /* HAVE_SO_TIMESTAMP */ } - dbgprintf("XXX: post CM loop\n"); } -# endif /* HAVE_SCM_CREDENTIALS */ - CHKiRet(SubmitMsg(pRcv, iRcvd, pLstn, cred)); + CHKiRet(SubmitMsg(pRcv, iRcvd, pLstn, cred, ts)); } else if(iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -786,8 +969,10 @@ CODESTARTwillRun listeners[0].ratelimitInterval = ratelimitIntervalSysSock; listeners[0].ratelimitBurst = ratelimitBurstSysSock; listeners[0].ratelimitSev = ratelimitSeveritySysSock; - listeners[0].bUseCreds = (bWritePidSysSock || ratelimitIntervalSysSock) ? 1 : 0; + listeners[0].bUseCreds = (bWritePidSysSock || ratelimitIntervalSysSock || bAnnotateSysSock) ? 1 : 0; listeners[0].bWritePid = bWritePidSysSock; + listeners[0].bAnnotate = bAnnotateSysSock; + listeners[0].bUseSysTimeStamp = bUseSysTimeStampSysSock; sd_fds = sd_listen_fds(0); if (sd_fds < 0) { @@ -830,7 +1015,6 @@ CODESTARTafterRun /* Clean-up files. */ for(i = startIndexUxLocalSockets; i < nfd; i++) if (listeners[i].sockName && listeners[i].fd != -1) { - /* If systemd passed us a socket it is systemd's job to clean it up. * Do not unlink it -- we will get same socket (node) from systemd * e.g. on restart again. @@ -900,6 +1084,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bUseFlowCtl = 0; bWritePid = 0; bWritePidSysSock = 0; + bUseSysTimeStamp = 1; + bUseSysTimeStampSysSock = 1; bCreatePath = DFLT_bCreatePath; ratelimitInterval = DFLT_ratelimitInterval; ratelimitIntervalSysSock = DFLT_ratelimitInterval; @@ -934,7 +1120,9 @@ CODEmodInit_QueryRegCFSLineHdlr listeners[0].fd = -1; listeners[0].bParseHost = 0; listeners[0].bUseCreds = 0; + listeners[0].bAnnotate = 0; listeners[0].bCreatePath = 0; + listeners[0].bUseSysTimeStamp = 1; /* initialize socket names */ for(i = 1 ; i < MAXFUNIX ; ++i) { @@ -962,10 +1150,14 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &pLogHostName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketflowcontrol", 0, eCmdHdlrBinary, NULL, &bUseFlowCtl, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketannotate", 0, eCmdHdlrBinary, + NULL, &bAnnotate, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketcreatepath", 0, eCmdHdlrBinary, NULL, &bCreatePath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bWritePid, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputunixlistensocketusesystimestamp", 0, eCmdHdlrBinary, + NULL, &bUseSysTimeStamp, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"addunixlistensocket", 0, eCmdHdlrGetWord, addLstnSocketName, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"imuxsockratelimitinterval", 0, eCmdHdlrInt, @@ -986,6 +1178,10 @@ CODEmodInit_QueryRegCFSLineHdlr setSystemLogTimestampIgnore, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketflowcontrol", 0, eCmdHdlrBinary, setSystemLogFlowControl, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusesystimestamp", 0, eCmdHdlrBinary, + NULL, &bUseSysTimeStampSysSock, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogsocketannotate", 0, eCmdHdlrBinary, + NULL, &bAnnotateSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogusepidfromsystem", 0, eCmdHdlrBinary, NULL, &bWritePidSysSock, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"systemlogratelimitinterval", 0, eCmdHdlrInt, diff --git a/plugins/mmsnmptrapd/mmsnmptrapd.c b/plugins/mmsnmptrapd/mmsnmptrapd.c index 767829d6..b78046ee 100644 --- a/plugins/mmsnmptrapd/mmsnmptrapd.c +++ b/plugins/mmsnmptrapd/mmsnmptrapd.c @@ -418,7 +418,7 @@ CODEmodInit_QueryRegCFSLineHdlr cs.pszTagName = NULL; cs.pszSeverityMapping = NULL; - CHKiRet(omsdRegCFSLineHdlr((uchar *)"mmsnmptrapdtag", 0, eCmdHdlrInt, + CHKiRet(omsdRegCFSLineHdlr((uchar *)"mmsnmptrapdtag", 0, eCmdHdlrGetWord, NULL, &cs.pszTagName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"mmsnmptrapdseveritymapping", 0, eCmdHdlrGetWord, NULL, &cs.pszSeverityMapping, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/omelasticsearch/Makefile.am b/plugins/omelasticsearch/Makefile.am new file mode 100644 index 00000000..a574c72f --- /dev/null +++ b/plugins/omelasticsearch/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omelasticsearch.la + +omelasticsearch_la_SOURCES = omelasticsearch.c +omelasticsearch_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omelasticsearch_la_LDFLAGS = -module -avoid-version +omelasticsearch_la_LIBADD = $(CURL_LIBS) + +EXTRA_DIST = diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c new file mode 100644 index 00000000..3bec1838 --- /dev/null +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -0,0 +1,270 @@ +/* omelasticsearch.c + * This is the http://www.elasticsearch.org/ output module. + * + * NOTE: read comments in module-template.h for more specifics! + * + * Copyright 2011 Nathan Scott. + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <curl/curl.h> +#include <curl/types.h> +#include <curl/easy.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <time.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "statsobj.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP + +/* internal structures */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(statsobj) + +statsobj_t *indexStats; +STATSCOUNTER_DEF(indexConFail, mutIndexConFail) +STATSCOUNTER_DEF(indexSubmit, mutIndexSubmit) +STATSCOUNTER_DEF(indexFailed, mutIndexFailed) +STATSCOUNTER_DEF(indexSuccess, mutIndexSuccess) + +/* REST API for elasticsearch hits this URL: + * http://<hostName>:<restPort>/<searchIndex>/<searchType> + */ +typedef struct curl_slist HEADER; +typedef struct _instanceData { + CURL *curlHandle; /* libcurl session handle */ + HEADER *postHeader; /* json POST request info */ +} instanceData; + +/* config variables */ +static int restPort = 9200; +static char *hostName = "localhost"; +static char *searchIndex = "system"; +static char *searchType = "events"; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +BEGINfreeInstance +CODESTARTfreeInstance + if (pData->postHeader) { + curl_slist_free_all(pData->postHeader); + pData->postHeader = NULL; + } + if (pData->curlHandle) { + curl_easy_cleanup(pData->curlHandle); + pData->curlHandle = NULL; + } +ENDfreeInstance + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +rsRetVal +curlPost(instanceData *instance, uchar *message) +{ + CURLcode code; + CURL *curl = instance->curlHandle; + int length = strlen((char *)message); + + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, length); + code = curl_easy_perform(curl); + switch (code) { + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_CONNECT: + case CURLE_WRITE_ERROR: + STATSCOUNTER_INC(indexConFail, mutIndexConFail); + return RS_RET_SUSPENDED; + default: + STATSCOUNTER_INC(indexSubmit, mutIndexSubmit); + return RS_RET_OK; + } +} + +BEGINdoAction +CODESTARTdoAction + CHKiRet(curlPost(pData, ppString[0])); +finalize_it: +ENDdoAction + +/* elasticsearch POST result string ... useful for debugging */ +size_t +curlResult(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + unsigned int i; + char *p = (char *)ptr; + char *jsonData = (char *)userdata; + static char ok[] = "{\"ok\":true,"; + + ASSERT(size == 1); + + if (size == 1 && + nmemb > sizeof(ok)-1 && + strncmp(p, ok, sizeof(ok)-1) == 0) { + STATSCOUNTER_INC(indexSuccess, mutIndexSuccess); + } else { + STATSCOUNTER_INC(indexFailed, mutIndexFailed); + if (Debug) { + DBGPRINTF("omelasticsearch request: %s\n", jsonData); + DBGPRINTF("omelasticsearch result: "); + for (i = 0; i < nmemb; i++) + DBGPRINTF("%c", p[i]); + DBGPRINTF("\n"); + } + } + return size * nmemb; +} + +static rsRetVal +curlSetup(instanceData *instance) +{ + char restURL[2048]; /* libcurl makes a copy, using the stack here is OK */ + HEADER *header; + CURL *handle; + + handle = curl_easy_init(); + if (handle == NULL) { + return RS_RET_OBJ_CREATION_FAILED; + } + + snprintf(restURL, sizeof(restURL)-1, "http://%s:%d/%s/%s", + hostName, restPort, searchIndex, searchType); + header = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8"); + + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult); + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header); + curl_easy_setopt(handle, CURLOPT_URL, restURL); + curl_easy_setopt(handle, CURLOPT_POST, 1); + + instance->curlHandle = handle; + instance->postHeader = header; + + DBGPRINTF("omelasticsearch setup, using REST URL: %s\n", restURL); + return RS_RET_OK; +} + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + if(strncmp((char*) p, ":omelasticsearch:", sizeof(":omelasticsearch:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + p += sizeof(":omelasticsearch:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (uchar*) " StdJSONFmt")); + + /* all good, we can now initialise our private data */ + CHKiRet(curlSetup(pData)); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINmodExit +CODESTARTmodExit + curl_global_cleanup(); + statsobj.Destruct(&indexStats); + objRelease(errmsg, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); +ENDmodExit + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + restPort = 9200; + hostName = "localhost"; + searchIndex = "system"; + searchType = "events"; + RETiRet; +} + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); + + /* register config file handlers */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchindex", 0, eCmdHdlrGetWord, NULL, &searchIndex, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchtype", 0, eCmdHdlrGetWord, NULL, &searchType, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchhost", 0, eCmdHdlrGetWord, NULL, &hostName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchport", 0, eCmdHdlrInt, NULL, &restPort, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + + if (curl_global_init(CURL_GLOBAL_ALL) != 0) { + errmsg.LogError(0, RS_RET_OBJ_CREATION_FAILED, "CURL fail. -elasticsearch indexing disabled"); + ABORT_FINALIZE(RS_RET_OBJ_CREATION_FAILED); + } + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&indexStats)); + CHKiRet(statsobj.SetName(indexStats, (uchar *)"elasticsearch")); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"connfail", + ctrType_IntCtr, &indexConFail)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"submits", + ctrType_IntCtr, &indexSubmit)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed", + ctrType_IntCtr, &indexFailed)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"success", + ctrType_IntCtr, &indexSuccess)); + CHKiRet(statsobj.ConstructFinalize(indexStats)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/plugins/omhdfs/javaenv.sh b/plugins/omhdfs/javaenv.sh new file mode 100644 index 00000000..d07a8473 --- /dev/null +++ b/plugins/omhdfs/javaenv.sh @@ -0,0 +1,14 @@ +# This is a sample file for environment settings on Fedora 13 +# that made me compile & run omhdfs. I really *hate* the way +# java uses environment variables... Hopefully this file will +# help building and testing omhdfs in the future (there is also +# some more information in the rsyslog wiki). +# rgerhards, 2011-03-11 +# this now works, but don't ask my why ;) +#export JAVA_HOME=/usr/java/jdk1.6.0_21/bin/java +export PATH=/usr/java/jdk1.6.0_21/bin:$PATH +export JAVA_INCLUDES="-I/usr/java/jdk1.6.0_21/include -I/usr/java/jdk1.6.0_21/include/linux" +export JAVA_LIBS="-L/usr/java/jdk1.6.0_21/jre/lib/i386 -ljava -ljvm -lverify" +export HADOOP_HOME=/usr/lib/hadoop +export CLASSPATH=/usr/lib/jvm/java-6-sun/lib:/usr/lib/hadoop/lib:/usr/lib/hadoop/hadoop-ant-0.20.2+320.jar:/usr/lib/hadoop/hadoop-core-0.20.2+320.jar:/usr/lib/hadoop/hadoop-examples-0.20.2+320.jar:/usr/lib/hadoop/hadoop-test-0.20.2+320.jar:/usr/lib/hadoop/hadoop-tools-0.20.2+320.jar/usr/lib/hadoop/lib/commons-cli-1.2.jar:/usr/lib/hadoop/lib/commons-codec-1.3.jar:/usr/lib/hadoop/lib/commons-el-1.0.jar:/usr/lib/hadoop/lib/commons-httpclient-3.0.1.jar:/usr/lib/hadoop/lib/commons-logging-1.0.4.jar:/usr/lib/hadoop/lib/commons-logging-api-1.0.4.jar:/usr/lib/hadoop/lib/commons-net-1.4.1.jar:/usr/lib/hadoop/lib/core-3.1.1.jar:/usr/lib/hadoop/lib/hadoop-fairscheduler-0.20.2+320.jar:/usr/lib/hadoop/lib/hadoop-scribe-log4j-0.20.2+320.jar:/usr/lib/hadoop/lib/hsqldb-1.8.0.10.jar:/usr/lib/hadoop/lib/hsqldb.jar:/usr/lib/hadoop/lib/jackson-core-asl-1.0.1.jar:/usr/lib/hadoop/lib/jackson-mapper-asl-1.0.1.jar:/usr/lib/hadoop/lib/jasper-compiler-5.5.12.jar:/usr/lib/hadoop/lib/jasper-runtime-5.5.12.jar:/usr/lib/hadoop/lib/jets3t-0.6.1.jar:/usr/lib/hadoop/lib/jetty-6.1.14.jar:/usr/lib/hadoop/lib/jetty-util-6.1.14.jar:/usr/lib/hadoop/lib/junit-4.5.jar:/usr/lib/hadoop/lib/kfs-0.2.2.jar:/usr/lib/hadoop/lib/libfb303.jar:/usr/lib/hadoop/lib/libthrift.jar:/usr/lib/hadoop/lib/log4j-1.2.15.jar:/usr/lib/hadoop/lib/mockito-all-1.8.2.jar:/usr/lib/hadoop/lib/mysql-connector-java-5.0.8-bin.jar:/usr/lib/hadoop/lib/oro-2.0.8.jar:/usr/lib/hadoop/lib/servlet-api-2.5-6.1.14.jar:/usr/lib/hadoop/lib/slf4j-api-1.4.3.jar:/usr/lib/hadoop/lib/slf4j-log4j12-1.4.3.jar:/usr/lib/hadoop/lib/xmlenc-0.52.jar:/etc/hadoop/conf +###export CLASSPATH="/usr/lib/hadoop/hadoop-0.20.2+320-ant.jar: /usr/lib/hadoop/hadoop-0.20.2+320-core.jar: /usr/lib/hadoop/hadoop-0.20.2+320-examples.jar: /usr/lib/hadoop/hadoop-0.20.2+320-test.jar: /usr/lib/hadoop/hadoop-0.20.2+320-tools.jar: /usr/lib/hadoop/hadoop-ant-0.20.2+320.jar: /usr/lib/hadoop/hadoop-core-0.20.2+320.jar: /usr/lib/hadoop/hadoop-examples-0.20.2+320.jar: /usr/lib/hadoop/hadoop-test-0.20.2+320.jar: /usr/lib/hadoop/hadoop-tools-0.20.2+320.jar:/usr/lib/hadoop/lib: /usr/lib/hadoop/lib/commons-cli-1.2.jar: /usr/lib/hadoop/lib/commons-codec-1.3.jar: /usr/lib/hadoop/lib/commons-el-1.0.jar: /usr/lib/hadoop/lib/commons-httpclient-3.0.1.jar: /usr/lib/hadoop/lib/commons-logging-1.0.4.jar: /usr/lib/hadoop/lib/commons-logging-api-1.0.4.jar: /usr/lib/hadoop/lib/commons-net-1.4.1.jar: /usr/lib/hadoop/lib/core-3.1.1.jar: /usr/lib/hadoop/lib/hadoop-fairscheduler-0.20.2+320.jar: /usr/lib/hadoop/lib/hadoop-scribe-log4j-0.20.2+320.jar: /usr/lib/hadoop/lib/hsqldb-1.8.0.10.jar: /usr/lib/hadoop/lib/hsqldb.jar: /usr/lib/hadoop/lib/jackson-core-asl-1.0.1.jar: /usr/lib/hadoop/lib/jackson-mapper-asl-1.0.1.jar: /usr/lib/hadoop/lib/jasper-compiler-5.5.12.jar: /usr/lib/hadoop/lib/jasper-runtime-5.5.12.jar: /usr/lib/hadoop/lib/jets3t-0.6.1.jar: /usr/lib/hadoop/lib/jetty-6.1.14.jar: /usr/lib/hadoop/lib/jetty-util-6.1.14.jar: /usr/lib/hadoop/lib/junit-4.5.jar: /usr/lib/hadoop/lib/kfs-0.2.2.jar: /usr/lib/hadoop/lib/libfb303.jar: /usr/lib/hadoop/lib/libthrift.jar: /usr/lib/hadoop/lib/log4j-1.2.15.jar: /usr/lib/hadoop/lib/mockito-all-1.8.2.jar: /usr/lib/hadoop/lib/mysql-connector-java-5.0.8-bin.jar: /usr/lib/hadoop/lib/oro-2.0.8.jar: /usr/lib/hadoop/lib/servlet-api-2.5-6.1.14.jar: /usr/lib/hadoop/lib/slf4j-api-1.4.3.jar: /usr/lib/hadoop/lib/slf4j-log4j12-1.4.3.jar: /usr/lib/hadoop/lib/xmlenc-0.52.jar:/etc/hadoop/conf:/usr/lib/hadoop/lib" diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index 8b72747f..76128a4e 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -80,6 +80,8 @@ typedef struct { typedef struct _instanceData { file_t *pFile; + uchar ioBuf[64*1024]; + unsigned offsBuf; } instanceData; /* forward definitions (down here, need data types) */ @@ -260,7 +262,8 @@ fileOpen(file_t *pFile) if(errno == ENOENT) { DBGPRINTF("omhdfs: ENOENT trying to append to '%s', now trying create\n", pFile->name); - pFile->fh = hdfsOpenFile(pFile->fs, (char*)pFile->name, O_WRONLY|O_CREAT, 0, 0, 0); + pFile->fh = hdfsOpenFile(pFile->fs, + (char*)pFile->name, O_WRONLY|O_CREAT, 0, 0, 0); } } if(pFile->fh == NULL) { @@ -275,12 +278,15 @@ finalize_it: } +/* Note: lenWrite is reset to zero on successful write! */ static inline rsRetVal -fileWrite(file_t *pFile, uchar *buf) +fileWrite(file_t *pFile, uchar *buf, size_t *lenWrite) { - size_t lenWrite; DEFiRet; + if(*lenWrite == 0) + FINALIZE; + if(pFile->nUsers > 1) d_pthread_mutex_lock(&pFile->mut); @@ -294,18 +300,18 @@ fileWrite(file_t *pFile, uchar *buf) } } - lenWrite = strlen((char*) buf); - tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, lenWrite); - if((unsigned) num_written_bytes != lenWrite) { - errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, "omhdfs: failed to write %s, expected %lu bytes, " - "written %lu\n", pFile->name, (unsigned long) lenWrite, +dbgprintf("XXXXX: omhdfs writing %u bytes\n", *lenWrite); + tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, *lenWrite); + if((unsigned) num_written_bytes != *lenWrite) { + errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, + "omhdfs: failed to write %s, expected %lu bytes, " + "written %lu\n", pFile->name, (unsigned long) *lenWrite, (unsigned long) num_written_bytes); ABORT_FINALIZE(RS_RET_SUSPENDED); } + *lenWrite = 0; finalize_it: - if(pFile->nUsers > 1) - d_pthread_mutex_unlock(&pFile->mut); RETiRet; } @@ -333,6 +339,40 @@ finalize_it: /* ---END FILE OBJECT---------------------------------------------------- */ +/* This adds data to the output buffer and performs an actual write + * if the new data does not fit into the buffer. Note that we never write + * partial data records. Other actions may write into the same file, and if + * we would write partial records, data could become severely mixed up. + * Note that we must check of some new data arrived is large than our + * buffer. In that case, the new data will written with its own + * write operation. + */ +static inline rsRetVal +addData(instanceData *pData, uchar *buf) +{ + unsigned len; + DEFiRet; + + len = strlen((char*)buf); + if(pData->offsBuf + len < sizeof(pData->ioBuf)) { + /* new data fits into remaining buffer */ + memcpy((char*) pData->ioBuf + pData->offsBuf, buf, len); + pData->offsBuf += len; + } else { +dbgprintf("XXXXX: not enough room, need to flush\n"); + CHKiRet(fileWrite(pData->pFile, pData->ioBuf, &pData->offsBuf)); + if(len >= sizeof(pData->ioBuf)) { + CHKiRet(fileWrite(pData->pFile, buf, &len)); + } else { + memcpy((char*) pData->ioBuf + pData->offsBuf, buf, len); + pData->offsBuf += len; + } + } + + iRet = RS_RET_DEFER_COMMIT; +finalize_it: + RETiRet; +} BEGINcreateInstance CODESTARTcreateInstance @@ -358,13 +398,31 @@ CODESTARTtryResume } ENDtryResume + +BEGINbeginTransaction +CODESTARTbeginTransaction +dbgprintf("omhdfs: beginTransaction\n"); +ENDbeginTransaction + + BEGINdoAction CODESTARTdoAction - DBGPRINTF("omuxsock: action to to write to %s\n", pData->pFile->name); - iRet = fileWrite(pData->pFile, ppString[0]); + DBGPRINTF("omhdfs: action to to write to %s\n", pData->pFile->name); + iRet = addData(pData, ppString[0]); +dbgprintf("omhdfs: done doAction\n"); ENDdoAction +BEGINendTransaction +CODESTARTendTransaction +dbgprintf("omhdfs: endTransaction\n"); + if(pData->offsBuf != 0) { + DBGPRINTF("omhdfs: data unwritten at end of transaction, persisting...\n"); + iRet = fileWrite(pData->pFile, pData->ioBuf, &pData->offsBuf); + } +ENDendTransaction + + BEGINparseSelectorAct file_t *pFile; int r; @@ -409,6 +467,7 @@ CODESTARTparseSelectorAct } fileObjAddUser(pFile); pData->pFile = pFile; + pData->offsBuf = 0; CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -455,6 +514,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ CODEqueryEtryPt_doHUP ENDqueryEtryPt @@ -472,5 +532,6 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsport", 0, eCmdHdlrInt, NULL, &hdfsPort, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"omhdfsdefaulttemplate", 0, eCmdHdlrGetWord, NULL, &dfltTplName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + DBGPRINTF("omhdfs: module compiled with rsyslog version %s.\n", VERSION); CODEmodInit_QueryRegCFSLineHdlr ENDmodInit diff --git a/runtime/conf.c b/runtime/conf.c index d731c786..cbba1790 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -399,6 +399,7 @@ processConfFile(uchar *pConfFile) uchar cbuf[CFGLNSIZ]; uchar *cline; int i; + rsRetVal localRet; int bHadAnError = 0; uchar *pszOrgLine = NULL; size_t lenLine; @@ -457,16 +458,20 @@ processConfFile(uchar *pConfFile) /* we now have the complete line, and are positioned at the first non-whitespace * character. So let's process it */ - if(cfline(cbuf, &pCurrRule) != RS_RET_OK) { + if((localRet = cfline(cbuf, &pCurrRule)) != RS_RET_OK) { /* we log a message, but otherwise ignore the error. After all, the next * line can be correct. -- rgerhards, 2007-08-02 */ uchar szErrLoc[MAXFNAME + 64]; - dbgprintf("config line NOT successfully processed\n"); + if(localRet != RS_RET_OK_WARN) { + dbgprintf("config line NOT successfully processed\n"); + bHadAnError = 1; + } snprintf((char*)szErrLoc, sizeof(szErrLoc) / sizeof(uchar), "%s, line %d", pConfFile, iLnNbr); - errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s:\"%s\"", (char*)szErrLoc, (char*)pszOrgLine); - bHadAnError = 1; + errmsg.LogError(0, NO_ERRCODE, "the last %s occured in %s:\"%s\"", + (localRet == RS_RET_OK_WARN) ? "warning" : "error", + (char*)szErrLoc, (char*)pszOrgLine); } } @@ -648,17 +653,14 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule for (bp=buf; *(bp+1); bp++) *bp=*(bp+1); *bp='\0'; - } - else { + } else { ignorepri = 0; } - if ( *buf == '=' ) - { + if ( *buf == '=' ) { singlpri = 1; pri = decodeSyslogName(&buf[1], syslogPriNames); } - else { - singlpri = 0; + else { singlpri = 0; pri = decodeSyslogName(buf, syslogPriNames); } @@ -686,17 +688,13 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule pRule->f_filterData.f_pmask[i] &= ~(1<<pri); else pRule->f_filterData.f_pmask[i] |= (1<<pri); - } - else - { + } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; else pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; - } - else - { + } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) pRule->f_filterData.f_pmask[i] &= ~(1<<i2); @@ -1082,6 +1080,7 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) omodStringRequest_t *pOMSR; action_t *pAction = NULL; void *pModData; + int bHadWarning = 0; ASSERT(p != NULL); ASSERT(ppAction != NULL); @@ -1097,6 +1096,10 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) pOMSR = NULL; iRet = pMod->mod.om.parseSelectorAct(p, &pModData, &pOMSR); dbgprintf("tried selector action for %s: %d\n", module.GetName(pMod), iRet); + if(iRet == RS_RET_OK_WARN) { + bHadWarning = 1; + iRet = RS_RET_OK; + } if(iRet == RS_RET_OK || iRet == RS_RET_SUSPENDED) { if((iRet = addAction(&pAction, pMod, pModData, pOMSR, (iRet == RS_RET_SUSPENDED)? 1 : 0)) == RS_RET_OK) { /* now check if the module is compatible with select features */ @@ -1125,6 +1128,8 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) } *ppAction = pAction; + if(iRet == RS_RET_OK && bHadWarning) + iRet = RS_RET_OK_WARN; RETiRet; } @@ -1139,6 +1144,8 @@ cflineClassic(uchar *p, rule_t **ppRule) { DEFiRet; action_t *pAction; + rsRetVal localRet; + int bHadWarning = 0; /* lines starting with '&' have no new filters and just add * new actions to the currently processed selector. @@ -1165,10 +1172,17 @@ cflineClassic(uchar *p, rule_t **ppRule) CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } - CHKiRet(cflineDoAction(&p, &pAction)); + localRet = cflineDoAction(&p, &pAction); + if(localRet == RS_RET_OK_WARN) { + bHadWarning = 1; + } else { + CHKiRet(localRet); + } CHKiRet(llAppend(&(*ppRule)->llActList, NULL, (void*) pAction)); finalize_it: + if(iRet == RS_RET_OK && bHadWarning) + iRet = RS_RET_OK_WARN; RETiRet; } diff --git a/runtime/datetime.c b/runtime/datetime.c index 679ce0b4..85cbab84 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -53,6 +53,47 @@ static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; /* ------------------------------ methods ------------------------------ */ +/** + * Convert struct timeval to syslog_time + */ +void +timeval2syslogTime(struct timeval *tp, struct syslogTime *t) +{ + struct tm *tm; + struct tm tmBuf; + long lBias; + + tm = localtime_r((time_t*) &(tp->tv_sec), &tmBuf); + + t->year = tm->tm_year + 1900; + t->month = tm->tm_mon + 1; + t->day = tm->tm_mday; + t->hour = tm->tm_hour; + t->minute = tm->tm_min; + t->second = tm->tm_sec; + t->secfrac = tp->tv_usec; + t->secfracPrecision = 6; + +# if __sun + /* Solaris uses a different method of exporting the time zone. + * It is UTC - localtime, which is the opposite sign of mins east of GMT. + */ + lBias = -(daylight ? altzone : timezone); +# elif defined(__hpux) + lBias = tz.tz_dsttime ? - tz.tz_minuteswest : 0; +# else + lBias = tm->tm_gmtoff; +# endif + if(lBias < 0) { + t->OffsetMode = '-'; + lBias *= -1; + } else + t->OffsetMode = '+'; + t->OffsetHour = lBias / 3600; + t->OffsetMinute = (lBias % 3600) / 60; + t->timeType = TIME_TYPE_RFC5424; /* we have a high precision timestamp */ +} + /** * Get the current date/time in the best resolution the operating * system has to offer (well, actually at most down to the milli- @@ -72,9 +113,6 @@ static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) { struct timeval tp; - struct tm *tm; - struct tm tmBuf; - long lBias; # if defined(__hpux) struct timezone tz; # endif @@ -91,37 +129,7 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) if(ttSeconds != NULL) *ttSeconds = tp.tv_sec; - tm = localtime_r((time_t*) &(tp.tv_sec), &tmBuf); - - t->year = tm->tm_year + 1900; - t->month = tm->tm_mon + 1; - t->day = tm->tm_mday; - t->hour = tm->tm_hour; - t->minute = tm->tm_min; - t->second = tm->tm_sec; - t->secfrac = tp.tv_usec; - t->secfracPrecision = 6; - -# if __sun - /* Solaris uses a different method of exporting the time zone. - * It is UTC - localtime, which is the opposite sign of mins east of GMT. - */ - lBias = -(daylight ? altzone : timezone); -# elif defined(__hpux) - lBias = tz.tz_dsttime ? - tz.tz_minuteswest : 0; -# else - lBias = tm->tm_gmtoff; -# endif - if(lBias < 0) - { - t->OffsetMode = '-'; - lBias *= -1; - } - else - t->OffsetMode = '+'; - t->OffsetHour = lBias / 3600; - t->OffsetMinute = (lBias % 3600) / 60; - t->timeType = TIME_TYPE_RFC5424; /* we have a high precision timestamp */ + timeval2syslogTime(&tp, t); } @@ -859,6 +867,7 @@ CODESTARTobjQueryInterface(datetime) */ pIf->getCurrTime = getCurrTime; pIf->GetTime = getTime; + pIf->timeval2syslogTime = timeval2syslogTime; pIf->ParseTIMESTAMP3339 = ParseTIMESTAMP3339; pIf->ParseTIMESTAMP3164 = ParseTIMESTAMP3164; pIf->formatTimestampToMySQL = formatTimestampToMySQL; diff --git a/runtime/datetime.h b/runtime/datetime.h index 7fcd273b..acf54df5 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -42,8 +42,10 @@ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); /* v3, 2009-11-12 */ time_t (*GetTime)(time_t *ttSeconds); + /* v6, 2011-06-20 */ + void (*timeval2syslogTime)(struct timeval *tp, struct syslogTime *t); ENDinterface(datetime) -#define datetimeCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define datetimeCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* interface changes: * 1 - initial version * 2 - not compatible to 1 - bugfix required ParseTIMESTAMP3164 to accept char ** as @@ -52,6 +54,7 @@ ENDinterface(datetime) * 3 - taken by v5 branch! * 4 - formatTimestamp3164 takes a third int parameter * 5 - merge of versions 3 + 4 (2010-03-09) + * 6 - see above */ /* prototypes */ diff --git a/runtime/module-template.h b/runtime/module-template.h index e21d6157..63bf9a10 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -275,7 +275,7 @@ static rsRetVal parseSelectorAct(uchar **pp, void **ppModData, omodStringRequest #define CODE_STD_FINALIZERparseSelectorAct \ finalize_it:\ - if(iRet == RS_RET_OK || iRet == RS_RET_SUSPENDED) {\ + if(iRet == RS_RET_OK || iRet == RS_RET_OK_WARN || iRet == RS_RET_SUSPENDED) {\ *ppModData = pData;\ *pp = p;\ } else {\ diff --git a/runtime/modules.c b/runtime/modules.c index 4541bddf..6a32b2e8 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -767,7 +767,6 @@ Load(uchar *pModName) DEFiRet; size_t iPathLen, iModNameLen; - uchar szPath[PATH_MAX]; uchar *pModNameCmp; int bHasExtension; void *pModHdlr, *pModInit; @@ -775,13 +774,25 @@ Load(uchar *pModName) uchar *pModDirCurr, *pModDirNext; int iLoadCnt; struct dlhandle_s *pHandle = NULL; +# ifdef PATH_MAX + uchar pathBuf[PATH_MAX+1]; +# else + uchar pathBuf[4096]; +# endif + uchar *pPathBuf = pathBuf; + size_t lenPathBuf = sizeof(pathBuf); assert(pModName != NULL); dbgprintf("Requested to load module '%s'\n", pModName); + iModNameLen = strlen((char*)pModName); + /* overhead for a full path is potentially 1 byte for a slash, + * three bytes for ".so" and one byte for '\0'. + */ +# define PATHBUF_OVERHEAD 1 + iModNameLen + 3 + 1 + pthread_mutex_lock(&mutLoadUnload); - iModNameLen = strlen((char *) pModName); if(iModNameLen > 3 && !strcmp((char *) pModName + iModNameLen - 3, ".so")) { iModNameLen -= 3; bHasExtension = TRUE; @@ -802,13 +813,19 @@ Load(uchar *pModName) pModDirNext = NULL; pModHdlr = NULL; iLoadCnt = 0; - do { - /* now build our load module name */ + do { /* now build our load module name */ if(*pModName == '/' || *pModName == '.') { - *szPath = '\0'; /* we do not need to append the path - its already in the module name */ + if(lenPathBuf < PATHBUF_OVERHEAD) { + if(pPathBuf != pathBuf) /* already malloc()ed memory? */ + free(pPathBuf); + /* we always alloc enough memory for everything we potentiall need to add */ + lenPathBuf = PATHBUF_OVERHEAD; + CHKmalloc(pPathBuf = malloc(sizeof(char)*lenPathBuf)); + } + *pPathBuf = '\0'; /* we do not need to append the path - its already in the module name */ iPathLen = 0; } else { - *szPath = '\0'; + *pPathBuf = '\0'; iPathLen = strlen((char *)pModDirCurr); pModDirNext = (uchar *)strchr((char *)pModDirCurr, ':'); @@ -821,30 +838,27 @@ Load(uchar *pModName) continue; } break; - } else if(iPathLen > sizeof(szPath) - 1) { - errmsg.LogError(0, NO_ERRCODE, "could not load module '%s', module path too long\n", pModName); - ABORT_FINALIZE(RS_RET_MODULE_LOAD_ERR_PATHLEN); + } else if(iPathLen > lenPathBuf - PATHBUF_OVERHEAD) { + if(pPathBuf != pathBuf) /* already malloc()ed memory? */ + free(pPathBuf); + /* we always alloc enough memory for everything we potentiall need to add */ + lenPathBuf = iPathLen + PATHBUF_OVERHEAD; + CHKmalloc(pPathBuf = malloc(sizeof(char)*lenPathBuf)); } - strncat((char *) szPath, (char *)pModDirCurr, iPathLen); - iPathLen = strlen((char*) szPath); + memcpy((char *) pPathBuf, (char *)pModDirCurr, iPathLen); + if((pPathBuf[iPathLen - 1] != '/')) { + /* we have space, made sure in previous check */ + pPathBuf[iPathLen++] = '/'; + } + pPathBuf[iPathLen] = '\0'; if(pModDirNext) pModDirCurr = pModDirNext + 1; - - if((szPath[iPathLen - 1] != '/')) { - if((iPathLen <= sizeof(szPath) - 2)) { - szPath[iPathLen++] = '/'; - szPath[iPathLen] = '\0'; - } else { - errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_PATHLEN, "could not load module '%s', path too long\n", pModName); - ABORT_FINALIZE(RS_RET_MODULE_LOAD_ERR_PATHLEN); - } - } } /* ... add actual name ... */ - strncat((char *) szPath, (char *) pModName, sizeof(szPath) - iPathLen - 1); + strncat((char *) pPathBuf, (char *) pModName, lenPathBuf - iPathLen - 1); /* now see if we have an extension and, if not, append ".so" */ if(!bHasExtension) { @@ -853,17 +867,12 @@ Load(uchar *pModName) * algo over time... -- rgerhards, 2008-03-05 */ /* ... so now add the extension */ - strncat((char *) szPath, ".so", sizeof(szPath) - strlen((char*) szPath) - 1); + strncat((char *) pPathBuf, ".so", lenPathBuf - strlen((char*) pPathBuf) - 1); iPathLen += 3; } - if(iPathLen + strlen((char*) pModName) >= sizeof(szPath)) { - errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_PATHLEN, "could not load module '%s', path too long\n", pModName); - ABORT_FINALIZE(RS_RET_MODULE_LOAD_ERR_PATHLEN); - } - /* complete load path constructed, so ... GO! */ - dbgprintf("loading module '%s'\n", szPath); + dbgprintf("loading module '%s'\n", pPathBuf); /* see if we have this one already */ for (pHandle = pHandles; pHandle; pHandle = pHandle->next) { @@ -875,7 +884,7 @@ Load(uchar *pModName) /* not found, try to dynamically link it */ if (!pModHdlr) { - pModHdlr = dlopen((char *) szPath, RTLD_NOW); + pModHdlr = dlopen((char *) pPathBuf, RTLD_NOW); } iLoadCnt++; @@ -884,25 +893,28 @@ Load(uchar *pModName) if(!pModHdlr) { if(iLoadCnt) { - errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_DLOPEN, "could not load module '%s', dlopen: %s\n", szPath, dlerror()); + errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_DLOPEN, "could not load module '%s', dlopen: %s\n", + pPathBuf, dlerror()); } else { - errmsg.LogError(0, NO_ERRCODE, "could not load module '%s', ModDir was '%s'\n", szPath, + errmsg.LogError(0, NO_ERRCODE, "could not load module '%s', ModDir was '%s'\n", pPathBuf, ((pModDir == NULL) ? _PATH_MODDIR : (char *)pModDir)); } ABORT_FINALIZE(RS_RET_MODULE_LOAD_ERR_DLOPEN); } if(!(pModInit = dlsym(pModHdlr, "modInit"))) { - errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_NO_INIT, "could not load module '%s', dlsym: %s\n", szPath, dlerror()); + errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_NO_INIT, "could not load module '%s', dlsym: %s\n", pPathBuf, dlerror()); dlclose(pModHdlr); ABORT_FINALIZE(RS_RET_MODULE_LOAD_ERR_NO_INIT); } if((iRet = doModInit(pModInit, (uchar*) pModName, pModHdlr)) != RS_RET_OK) { - errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_INIT_FAILED, "could not load module '%s', rsyslog error %d\n", szPath, iRet); + errmsg.LogError(0, RS_RET_MODULE_LOAD_ERR_INIT_FAILED, "could not load module '%s', rsyslog error %d\n", pPathBuf, iRet); dlclose(pModHdlr); ABORT_FINALIZE(RS_RET_MODULE_LOAD_ERR_INIT_FAILED); } finalize_it: + if(pPathBuf != pathBuf) /* used malloc()ed memory? */ + free(pPathBuf); pthread_mutex_unlock(&mutLoadUnload); RETiRet; } diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 036e8290..e1192aaf 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -29,7 +29,9 @@ #include <string.h> #include <gnutls/gnutls.h> #include <gnutls/x509.h> -#include <gcrypt.h> +#if GNUTLS_VERSION_NUMBER <= 0x020b00 +# include <gcrypt.h> +#endif #include <errno.h> #include <sys/stat.h> #include <unistd.h> @@ -54,7 +56,9 @@ #define CRLFILE "crl.pem" +#if GNUTLS_VERSION_NUMBER <= 0x020b00 GCRY_THREAD_OPTION_PTHREAD_IMPL; +#endif MODULE_TYPE_LIB MODULE_TYPE_KEEP @@ -567,7 +571,9 @@ gtlsGlblInit(void) DEFiRet; /* gcry_control must be called first, so that the thread system is correctly set up */ + #if GNUTLS_VERSION_NUMBER <= 0x020b00 gcry_control (GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); + #endif CHKgnutls(gnutls_global_init()); /* X509 stuff */ diff --git a/runtime/parser.c b/runtime/parser.c index b385c54b..300db1e0 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -60,6 +60,7 @@ DEFobjCurrIf(ruleset) /* config data */ static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bSpaceLFOnRcv = 0; /* replace newlines with spaces on reception: 0 - no, 1 - yes */ static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */ static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ @@ -354,9 +355,13 @@ SanitizeMsg(msg_t *pMsg) int bNeedSanitize = 0; for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { if(iscntrl(pszMsg[iSrc])) { + if(bSpaceLFOnRcv && pszMsg[iSrc] == '\n') + pszMsg[iSrc] = ' '; + else if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { bNeedSanitize = 1; - break; + if (!bSpaceLFOnRcv) + break; } } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) { bNeedSanitize = 1; @@ -645,6 +650,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus { cCCEscapeChar = '#'; bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bSpaceLFOnRcv = 0; bEscape8BitChars = 0; /* default is to escape control characters */ bEscapeTab = 1; /* default is to escape control characters */ bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ @@ -698,6 +704,7 @@ BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"spacelfonreceive", 0, eCmdHdlrBinary, NULL, &bSpaceLFOnRcv, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); diff --git a/runtime/queue.c b/runtime/queue.c index e97d78e8..56d05571 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -83,6 +83,11 @@ static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti); static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti); static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub); static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub); +static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr); +static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis); +static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis); +static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis); +static rsRetVal qDestructDisk(qqueue_t *pThis); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -592,6 +597,47 @@ static rsRetVal qDelLinkedList(qqueue_t *pThis) /* -------------------- disk -------------------- */ +/* The following function is used to "save" ourself from being killed by + * a fatally failed disk queue. A fatal failure is, for example, if no + * data can be read or written. In that case, the disk support is disabled, + * with all on-disk structures kept as-is as much as possible. Instead, the + * queue is switched to direct mode, so that at least + * some processing can happen. Of course, this may still have lots of + * undesired side-effects, but is probably better than aborting the + * syslogd. Note that this function *must* succeed in one way or another, as + * we can not recover from failure here. But it may emit different return + * states, which can trigger different processing in the higher layers. + * rgerhards, 2011-05-03 + */ +static inline rsRetVal +queueSwitchToEmergencyMode(qqueue_t *pThis, rsRetVal initiatingError) +{ + pThis->iQueueSize = 0; + pThis->nLogDeq = 0; + qDestructDisk(pThis); /* free disk structures */ + + pThis->qType = QUEUETYPE_DIRECT; + pThis->qConstruct = qConstructDirect; + pThis->qDestruct = qDestructDirect; + pThis->qAdd = qAddDirect; + pThis->qDel = qDelDirect; + pThis->MultiEnq = qqueueMultiEnqObjDirect; + if(pThis->pqParent != NULL) { + DBGOPRINT((obj_t*) pThis, "DA queue is in emergency mode, disabling DA in parent\n"); + pThis->pqParent->bIsDA = 0; + pThis->pqParent->pqDA = NULL; + /* This may have undesired side effects, not sure if I really evaluated + * all. So you know where to look at if you come to this point during + * troubleshooting ;) -- rgerhards, 2011-05-03 + */ + } + + errmsg.LogError(0, initiatingError, "fatal error on disk queue '%s', emergency switch to direct mode", + obj.GetName((obj_t*) pThis)); + return RS_RET_ERR_QUEUE_EMERGENCY; +} + + static rsRetVal qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pThis) { @@ -794,10 +840,7 @@ finalize_it: static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr) { DEFiRet; - - CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL)); - -finalize_it: + iRet = obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL); RETiRet; } @@ -1694,7 +1737,18 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti) ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(DequeueForConsumer(pThis, pWti)); + iRet = DequeueForConsumer(pThis, pWti); + if(iRet == RS_RET_FILE_NOT_FOUND) { + /* This is a fatal condition and means the queue is almost unusable */ + d_pthread_mutex_unlock(pThis->mut); + DBGOPRINT((obj_t*) pThis, "got 'file not found' error %d, queue defunct\n", iRet); + iRet = queueSwitchToEmergencyMode(pThis, iRet); + // TODO: think about what to return as iRet -- keep RS_RET_FILE_NOT_FOUND? + d_pthread_mutex_lock(pThis->mut); + } + if (iRet != RS_RET_OK) { + FINALIZE; + } /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ d_pthread_mutex_unlock(pThis->mut); @@ -1788,7 +1842,6 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) { DEFiRet; -//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis)); if(pThis->bEnqOnly) { iRet = RS_RET_TERMINATE_WHEN_IDLE; } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 69b3c8d1..06438081 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -343,8 +343,10 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */ RS_RET_ERR_WRKDIR = -2181, /**< problems with the rsyslog working directory */ RS_RET_WRN_WRKDIR = -2182, /**< correctable problems with the rsyslog working directory */ + RS_RET_ERR_QUEUE_EMERGENCY = -2183, /**< some fatal error caused queue to switch to emergency mode */ RS_RET_OUTDATED_STMT = -2184, /**< some outdated statement/functionality is being used in conf file */ RS_RET_MISSING_WHITESPACE = -2185, /**< whitespace is missing in some config construct */ + RS_RET_OK_WARN = -2186, /**< config part: everything was OK, but a warning message was emitted */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index 0de18e7f..8310e832 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -765,7 +765,7 @@ static rsRetVal SetKeepAlive(strmsrv_t *pThis, int iVal) { DEFiRet; - dbgprintf("keep-alive set to %d\n", iVal); + dbgprintf("strmsrv: keep-alive set to %d\n", iVal); pThis->bUseKeepAlive = iVal; RETiRet; } diff --git a/runtime/wti.c b/runtime/wti.c index 0b85c366..9e781341 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -312,7 +312,10 @@ wtiWorker(wti_t *pThis) */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); - if(localRet == RS_RET_IDLE) { + if(localRet == RS_RET_ERR_QUEUE_EMERGENCY) { + d_pthread_mutex_unlock(pWtp->pmutUsr); + break; /* end of loop */ + } else if(localRet == RS_RET_IDLE) { if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { d_pthread_mutex_unlock(pWtp->pmutUsr); break; /* end of loop */ diff --git a/tcps_sess.c b/tcps_sess.c index 854d3742..921e05f2 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -253,7 +253,8 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); MsgSetInputName(pMsg, pThis->pLstnInfo->pInputName); - MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); + MsgSetFlowControlType(pMsg, pThis->pSrv->bUseFlowControl + ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; MsgSetRcvFrom(pMsg, pThis->fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); @@ -410,6 +410,10 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); } + if(pThis->bUseKeepAlive) { + CHKiRet(netstrm.EnableKeepAlive(pNewStrm)); + } + /* we found a free spot and can construct our session object */ CHKiRet(tcps_sess.Construct(&pSess)); CHKiRet(tcps_sess.SetTcpsrv(pSess, pThis)); @@ -734,6 +738,7 @@ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macr pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; pThis->bDisableLFDelim = 0; pThis->OnMsgReceive = NULL; + pThis->bUseFlowControl = 1; ENDobjConstruct(tcpsrv) @@ -880,6 +885,15 @@ SetUsrP(tcpsrv_t *pThis, void *pUsr) } static rsRetVal +SetKeepAlive(tcpsrv_t *pThis, int iVal) +{ + DEFiRet; + dbgprintf("tcpsrv: keep-alive set to %d\n", iVal); + pThis->bUseKeepAlive = iVal; + RETiRet; +} + +static rsRetVal SetOnMsgReceive(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)) { DEFiRet; @@ -1009,6 +1023,18 @@ SetLstnMax(tcpsrv_t *pThis, int iMax) } +/* set if flow control shall be supported + */ +static rsRetVal +SetUseFlowControl(tcpsrv_t *pThis, int bUseFlowControl) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->bUseFlowControl = bUseFlowControl; + RETiRet; +} + + /* set max number of sessions * this must be called before ConstructFinalize, or it will have no effect! * rgerhards, 2009-04-09 @@ -1046,11 +1072,13 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->create_tcp_socket = create_tcp_socket; pIf->Run = Run; + pIf->SetKeepAlive = SetKeepAlive; pIf->SetUsrP = SetUsrP; pIf->SetInputName = SetInputName; pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; pIf->SetbDisableLFDelim = SetbDisableLFDelim; pIf->SetSessMax = SetSessMax; + pIf->SetUseFlowControl = SetUseFlowControl; pIf->SetLstnMax = SetLstnMax; pIf->SetDrvrMode = SetDrvrMode; pIf->SetDrvrAuthMode = SetDrvrAuthMode; @@ -50,6 +50,7 @@ struct tcpLstnPortList_s { /* the tcpsrv object */ struct tcpsrv_s { BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ + int bUseKeepAlive; /**< use socket layer KEEPALIVE handling? */ netstrms_t *pNS; /**< pointer to network stream subsystem */ int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ @@ -58,6 +59,7 @@ struct tcpsrv_s { permittedPeers_t *pPermPeers;/**< driver's permitted peers */ sbool bEmitMsgOnClose; /**< emit an informational message when the remote peer closes connection */ sbool bUsingEPoll; /**< are we in epoll mode (means we do not need to keep track of sessions!) */ + sbool bUseFlowControl; /**< use flow control (make light delayable) */ int iLstnCurr; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ @@ -123,8 +125,12 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetNotificationOnRemoteClose)(tcpsrv_t *pThis, int bNewVal); /* 2009-10-01 */ /* added v9 -- rgerhards, 2010-03-01 */ rsRetVal (*SetbDisableLFDelim)(tcpsrv_t*, int); + /* added v10 -- rgerhards, 2011-04-01 */ + rsRetVal (*SetUseFlowControl)(tcpsrv_t*, int); + /* added v11 -- rgerhards, 2011-05-09 */ + rsRetVal (*SetKeepAlive)(tcpsrv_t*, int); ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 11 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 @@ -53,7 +53,12 @@ static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ - +enum { + NO_ESCAPE = 0, + SQL_ESCAPE, + STDSQL_ESCAPE, + JSON_ESCAPE, +}; /* helper to tplToString and strgen's, extends buffer */ #define ALLOC_INC 128 @@ -123,10 +128,12 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * * but they are handled in this way because of legacy (don't break any * existing thing). */ - if(pTpl->optFormatForSQL == 1) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); - else if(pTpl->optFormatForSQL == 2) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); + if(pTpl->optFormatEscape == SQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, SQL_ESCAPE); + else if(pTpl->optFormatEscape == JSON_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, JSON_ESCAPE); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, STDSQL_ESCAPE); } /* got source, now copy over */ if(iLenVal > 0) { /* may be zero depending on property */ @@ -213,27 +220,29 @@ finalize_it: } -/* Helper to doSQLEscape. This is called if doSQLEscape +/* Helper to doEscape. This is called if doEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can * NOT simply return the unmodified string because this * may cause SQL injection. But we also can not simply * abort the run, this would be a DoS. I think an appropriate - * measure is to remove the dangerous \' characters. We + * measure is to remove the dangerous \' characters (SQL). We * replace them by \", which will break the message and * signatures eventually present - but this is the * best thing we can do now (or does anybody * have a better idea?). rgerhards 2004-11-23 - * added support for "escapeMode" (so doSQLEscape for details). - * if mode = 1, then backslashes are changed to slashes. + * added support for escape mode (see doEscape for details). + * if mode = SQL_ESCAPE, then backslashes are changed to slashes. * rgerhards 2005-09-22 */ -static void doSQLEmergencyEscape(register uchar *p, int escapeMode) +static void doEmergencyEscape(register uchar *p, int mode) { while(*p) { - if(*p == '\'') + if((mode == SQL_ESCAPE||mode == STDSQL_ESCAPE) && *p == '\'') *p = '"'; - else if((escapeMode == 1) && (*p == '\\')) + else if((mode == JSON_ESCAPE) && *p == '"') + *p = '\''; + else if((mode == SQL_ESCAPE) && *p == '\\') *p = '/'; ++p; } @@ -258,14 +267,16 @@ static void doSQLEmergencyEscape(register uchar *p, int escapeMode) * smartness depends on config settings. So we add a new option to this * function that allows the caller to select if they want to standard or * "smart" encoding ;) - * new parameter escapeMode is 0 - standard sql, 1 - "smart" engines + * -- + * Parameter "mode" is STDSQL_ESCAPE, SQL_ESCAPE "smart" SQL engines, or + * JSON_ESCAPE for everyone requiring escaped JSON (e.g. ElasticSearch). * 2005-09-22 rgerhards */ rsRetVal -doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode) +doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int mode) { DEFiRet; - uchar *p; + uchar *p = NULL; int iLen; cstr_t *pStrB = NULL; uchar *pszGenerated; @@ -276,26 +287,32 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM assert(pbMustBeFreed != NULL); /* first check if we need to do anything at all... */ - if(escapeMode == 0) + if(mode == STDSQL_ESCAPE) for(p = *pp ; *p && *p != '\'' ; ++p) ; - else + else if(mode == SQL_ESCAPE) for(p = *pp ; *p && *p != '\'' && *p != '\\' ; ++p) ; + else if(mode == JSON_ESCAPE) + for(p = *pp ; *p && *p != '"' ; ++p) + ; /* when we get out of the loop, we are either at the - * string terminator or the first \'. */ - if(*p == '\0') + * string terminator or the first character to escape */ + if(p && *p == '\0') FINALIZE; /* nothing to do in this case! */ p = *pp; iLen = *pLen; CHKiRet(cstrConstruct(&pStrB)); - + while(*p) { - if(*p == '\'') { - CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + if((mode == SQL_ESCAPE || mode == STDSQL_ESCAPE) && *p == '\'') { + CHKiRet(cstrAppendChar(pStrB, (mode == STDSQL_ESCAPE) ? '\'' : '\\')); + iLen++; /* reflect the extra character */ + } else if((mode == SQL_ESCAPE) && *p == '\\') { + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ - } else if((escapeMode == 1) && (*p == '\\')) { + } else if((mode == JSON_ESCAPE) && *p == '"') { CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } @@ -314,7 +331,7 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM finalize_it: if(iRet != RS_RET_OK) { - doSQLEmergencyEscape(*pp, escapeMode); + doEmergencyEscape(*pp, mode); if(pStrB != NULL) cstrDestruct(&pStrB); } @@ -883,11 +900,14 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) * acknowledged implementing the option. -- rgerhards, 2011-03-21 */ if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) { - pTpl->optFormatForSQL = 2; - DBGPRINTF("strgen suports the stdsql option\n"); + pTpl->optFormatEscape = STDSQL_ESCAPE; + DBGPRINTF("strgen supports the stdsql option\n"); } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) { - pTpl->optFormatForSQL = 1; - DBGPRINTF("strgen suports the sql option\n"); + pTpl->optFormatEscape = SQL_ESCAPE; + DBGPRINTF("strgen supports the sql option\n"); + } else if(lenMod > 4 && !strcasecmp((char*) szMod+ lenMod - 4, ",json")) { + pTpl->optFormatEscape = JSON_ESCAPE; + DBGPRINTF("strgen supports the json option\n"); } finalize_it: @@ -1015,11 +1035,13 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) * it anyhow... ;) rgerhards 2004-11-22 */ if(!strcmp(optBuf, "stdsql")) { - pTpl->optFormatForSQL = 2; + pTpl->optFormatEscape = STDSQL_ESCAPE; + } else if(!strcmp(optBuf, "json")) { + pTpl->optFormatEscape = JSON_ESCAPE; } else if(!strcmp(optBuf, "sql")) { - pTpl->optFormatForSQL = 1; + pTpl->optFormatEscape = SQL_ESCAPE; } else if(!strcmp(optBuf, "nosql")) { - pTpl->optFormatForSQL = 0; + pTpl->optFormatEscape = NO_ESCAPE; } else { dbgprintf("Invalid option '%s' ignored.\n", optBuf); } @@ -1180,9 +1202,11 @@ void tplPrintList(void) pTpl = tplRoot; while(pTpl != NULL) { dbgprintf("Template: Name='%s' ", pTpl->pszName == NULL? "NULL" : pTpl->pszName); - if(pTpl->optFormatForSQL == 1) + if(pTpl->optFormatEscape == SQL_ESCAPE) dbgprintf("[SQL-Format (MySQL)] "); - else if(pTpl->optFormatForSQL == 2) + else if(pTpl->optFormatEscape == JSON_ESCAPE) + dbgprintf("[JSON-Escaped Format] "); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) dbgprintf("[SQL-Format (standard SQL)] "); dbgprintf("\n"); pTpe = pTpl->pEntryRoot; @@ -36,9 +36,10 @@ struct template { int tpenElements; /* number of elements in templateEntry list */ struct templateEntry *pEntryRoot; struct templateEntry *pEntryLast; - char optFormatForSQL; /* in text fields, 0 - do not escape, - * 1 - escape quotes by double quotes, - * 2 - escape "the MySQL way" + char optFormatEscape; /* in text fields, 0 - do not escape, + * 1 - escape "the MySQL way" + * 2 - escape quotes by double quotes, + * 3 - escape double quotes for JSON. */ /* following are options. All are 0/1 defined (either on or off). * we use chars because they are faster than bit fields and smaller @@ -130,7 +131,7 @@ rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); */ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); -rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); +rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); diff --git a/tests/inputfilegen.c b/tests/inputfilegen.c index 26fb79af..0ff8d049 100644 --- a/tests/inputfilegen.c +++ b/tests/inputfilegen.c @@ -1,5 +1,6 @@ /* generate an input file suitable for use by the testbench * Copyright (C) 2011 by Rainer Gerhards and Adiscon GmbH. + * usage: ./inputfilegen num-lines > file * Part of rsyslog, licensed under GPLv3 */ #include <stdio.h> diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 8485acbb..8a34f06f 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -91,8 +91,10 @@ #include <errno.h> #ifdef ENABLE_GNUTLS # include <gnutls/gnutls.h> -# include <gcrypt.h> +# if GNUTLS_VERSION_NUMBER <= 0x020b00 +# include <gcrypt.h> GCRY_THREAD_OPTION_PTHREAD_IMPL; +# endif #endif #define EXIT_FAILURE 1 @@ -707,7 +709,9 @@ initTLS(void) int r; /* order of gcry_control and gnutls_global_init matters! */ + #if GNUTLS_VERSION_NUMBER <= 0x020b00 gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); + #endif gnutls_global_init(); /* set debug mode, if so required by the options */ if(tlsLogLevel > 0) { diff --git a/tests/testsuites/imfile-basic.conf b/tests/testsuites/imfile-basic.conf index 9fb9b5ca..59b109a6 100644 --- a/tests/testsuites/imfile-basic.conf +++ b/tests/testsuites/imfile-basic.conf @@ -6,6 +6,7 @@ $InputFileTag file: $InputFileStateFile stat-file1 $InputFileSeverity error $InputFileFacility local7 +$InputFileMaxLinesAtOnce 100000 $InputRunFileMonitor $template outfmt,"%msg:F,58:2%\n" diff --git a/tools/omfile.c b/tools/omfile.c index fbd263c7..b94acc6d 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -122,13 +122,11 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; #define USE_ASYNCWRITER_DFLT 0 /* default buffer use async writer */ #define FLUSHONTX_DFLT 1 /* default for flush on TX end */ -#define DFLT_bForceChown 0 /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ static int fCreateMode = 0644; /* mode to use when creating files */ static int fDirCreateMode = 0700; /* mode to use when creating files */ static int bFailOnChown; /* fail if chown fails? */ -static int bForceChown = DFLT_bForceChown; /* Force chown() on existing files? */ static uid_t fileUID; /* UID to be used for newly created files */ static uid_t fileGID; /* GID to be used for newly created files */ static uid_t dirUID; /* UID to be used for newly created directories */ @@ -153,7 +151,6 @@ typedef struct _instanceData { int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ - sbool bForceChown; /* force chown() on existing files? */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -200,7 +197,6 @@ CODESTARTdbgPrintInstInfo dbgprintf("\tfile cache size=%d\n", pData->iDynaFileCacheSize); dbgprintf("\tcreate directories: %s\n", pData->bCreateDirs ? "yes" : "no"); dbgprintf("\tfile owner %d, group %d\n", (int) pData->fileUID, (int) pData->fileGID); - dbgprintf("\tforce chown() for all files: %s\n", pData->bForceChown ? "yes" : "no"); dbgprintf("\tdirectory owner %d, group %d\n", (int) pData->dirUID, (int) pData->dirGID); dbgprintf("\tdir create mode 0%3.3o, file create mode 0%3.3o\n", pData->fDirCreateMode, pData->fCreateMode); @@ -239,6 +235,14 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal) } +rsRetVal goneAway(void __attribute__((unused)) *pVal, + int __attribute__((unused)) iNewVal) +{ + errmsg.LogError(0, RS_RET_ERR, "directive $omfileForceChown is no longer supported"); + return RS_RET_ERR; +} + + /* Helper to cfline(). Parses a output channel name up until the first * comma and then looks for the template specifier. Tries * to find that template. Maps the output channel to the @@ -389,22 +393,7 @@ prepareFile(instanceData *pData, uchar *newFileName) DEFiRet; pData->pStrm = NULL; - if(access((char*)newFileName, F_OK) == 0) { - if(pData->bForceChown) { - /* Try to fix wrong ownership set by someone else. Note that this code - * will no longer work once we have made the $PrivDrop code fully secure. - * This change is based on an idea of Michael Terry, provided as part of - * the effort to make rsyslogd the Ubuntu default syslogd. - * rgerhards, 2009-09-11 - */ - if(chown((char*)newFileName, pData->fileUID, pData->fileGID) != 0) { - if(pData->bFailOnChown) { - int eSave = errno; - errno = eSave; - } - } - } - } else { + if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. @@ -424,7 +413,7 @@ prepareFile(instanceData *pData, uchar *newFileName) pData->fCreateMode); if(fd != -1) { /* check and set uid/gid */ - if(pData->bForceChown || pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { + if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ if(fchown(fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { @@ -654,6 +643,9 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) } else { /* "regular", non-dynafile */ if(pData->pStrm == NULL) { CHKiRet(prepareFile(pData, pData->f_fname)); + if(pData->pStrm == NULL) { + errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); + } } } @@ -811,7 +803,6 @@ CODESTARTparseSelectorAct pData->fDirCreateMode = fDirCreateMode; pData->bCreateDirs = bCreateDirs; pData->bFailOnChown = bFailOnChown; - pData->bForceChown = bForceChown; pData->fileUID = fileUID; pData->fileGID = fileGID; pData->dirUID = dirUID; @@ -821,18 +812,6 @@ CODESTARTparseSelectorAct pData->iIOBufSize = (int) iIOBufSize; pData->iFlushInterval = iFlushInterval; pData->bUseAsyncWriter = bUseAsyncWriter; - - if(pData->bDynamicName == 0) { - /* try open and emit error message if not possible. At this stage, we ignore the - * return value of prepareFile, this is taken care of in later steps. - */ - prepareFile(pData, pData->f_fname); - - if(pData->pStrm == NULL) { - DBGPRINTF("Error opening log file: %s\n", pData->f_fname); - errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could not open output file '%s'", pData->f_fname); - } - } CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -847,7 +826,6 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a dirUID = -1; dirGID = -1; bFailOnChown = 1; - bForceChown = DFLT_bForceChown; iDynaFileCacheSize = 10; fCreateMode = 0644; fDirCreateMode = 0700; @@ -922,7 +900,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"filecreatemode", 0, eCmdHdlrFileCreateMode, NULL, &fCreateMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"createdirs", 0, eCmdHdlrBinary, NULL, &bCreateDirs, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"failonchownfailure", 0, eCmdHdlrBinary, NULL, &bFailOnChown, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileForceChown", 0, eCmdHdlrBinary, NULL, &bForceChown, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileforcechown", 0, eCmdHdlrBinary, goneAway, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionfileenablesync", 0, eCmdHdlrBinary, NULL, &bEnableSync, STD_LOADABLE_MODULE_ID)); CHKiRet(regCfSysLineHdlr((uchar *)"actionfiledefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszFileDfltTplName, NULL)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); diff --git a/tools/omfwd.c b/tools/omfwd.c index 10cce0e2..b456db17 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -4,30 +4,26 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c) - * This file is under development and has not yet arrived at being fully - * self-contained and a real object. So far, it is mostly an excerpt - * of the "old" message code without any modifications. However, it - * helps to have things at the right place one we go to the meat of it. + * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c, which + * at the time of rsyslog fork from sysklogd was under BSD license) * - * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2012 Adiscon GmbH. * * This file is part of rsyslog. * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "config.h" #include "rsyslog.h" @@ -131,22 +127,6 @@ pData->bIsConnected = 0; // TODO: remove this variable altogether } -/* get the syslog forward port from selector_t. The passed in - * struct must be one that is setup for forwarding. - * rgerhards, 2007-06-28 - * We may change the implementation to try to lookup the port - * if it is unspecified. So far, we use the IANA default auf 514. - */ -static char *getFwdPt(instanceData *pData) -{ - assert(pData != NULL); - if(pData->port == NULL) - return("514"); - else - return(pData->port); -} - - /* destruct the TCP helper objects * This, for example, is needed after something went wrong. * This function is void because it "can not" fail. @@ -345,7 +325,7 @@ static rsRetVal TCPSendInit(void *pvData) } /* params set, now connect */ CHKiRet(netstrm.Connect(pData->pNetstrm, glbl.GetDefPFFamily(), - (uchar*)getFwdPt(pData), (uchar*)pData->f_hname)); + (uchar*)pData->port, (uchar*)pData->f_hname)); } finalize_it: @@ -378,9 +358,9 @@ static rsRetVal doTryResume(instanceData *pData) hints.ai_flags = AI_NUMERICSERV; hints.ai_family = glbl.GetDefPFFamily(); hints.ai_socktype = SOCK_DGRAM; - if((iErr = (getaddrinfo(pData->f_hname, getFwdPt(pData), &hints, &res))) != 0) { + if((iErr = (getaddrinfo(pData->f_hname, pData->port, &hints, &res))) != 0) { dbgprintf("could not get addrinfo for hostname '%s':'%s': %d%s\n", - pData->f_hname, getFwdPt(pData), iErr, gai_strerror(iErr)); + pData->f_hname, pData->port, iErr, gai_strerror(iErr)); ABORT_FINALIZE(RS_RET_SUSPENDED); } dbgprintf("%s found, resuming.\n", pData->f_hname); @@ -412,15 +392,18 @@ CODESTARTtryResume ENDtryResume BEGINdoAction - char *psz = NULL; /* temporary buffering */ + char *psz; /* temporary buffering */ register unsigned l; int iMaxLine; +# ifdef USE_NETZIP + Bytef *out = NULL; /* for compression */ +# endif CODESTARTdoAction CHKiRet(doTryResume(pData)); iMaxLine = glbl.GetMaxLine(); - dbgprintf(" %s:%s/%s\n", pData->f_hname, getFwdPt(pData), + dbgprintf(" %s:%s/%s\n", pData->f_hname, pData->port, pData->protocol == FORW_UDP ? "udp" : "tcp"); psz = (char*) ppString[0]; @@ -438,7 +421,6 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ if(pData->compressionLevel && (l > CONF_MIN_SIZE_FOR_COMPRESS)) { - Bytef *out; uLongf destLen = iMaxLine + iMaxLine/100 +12; /* recommended value from zlib doc */ uLong srcLen = l; int ret; @@ -459,14 +441,11 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ dbgprintf("Compression failed, sending uncompressed message\n"); - free(out); } else if(destLen+1 < l) { /* only use compression if there is a gain in using it! */ dbgprintf("there is gain in compression, so we do it\n"); psz = (char*) out; l = destLen + 1; /* take care for the "z" at message start! */ - } else { - free(out); } ++destLen; } @@ -488,10 +467,7 @@ CODESTARTdoAction } finalize_it: # ifdef USE_NETZIP - if((psz != NULL) && (psz != (char*) ppString[0])) { - /* we need to free temporary buffer, alloced above - Naoya Nakazawa, 2010-01-11 */ - free(psz); - } + free(out); /* is NULL if it was never used... */ # endif ENDdoAction @@ -630,12 +606,16 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) if(pData->port == NULL) { errmsg.LogError(0, NO_ERRCODE, "Could not get memory to store syslog forwarding port, " "using default port, results may not be what you intend\n"); - /* we leave f_forw.port set to NULL, this is then handled by getFwdPt(). */ + /* we leave f_forw.port set to NULL, this is then handled below */ } else { memcpy(pData->port, tmp, i); *(pData->port + i) = '\0'; } } + /* check if no port is set. If so, we use the IANA-assigned port of 514 */ + if(pData->port == NULL) { + CHKmalloc(pData->port = strdup("514")); + } /* now skip to template */ while(*p && *p != ';' && *p != '#' && !isspace((int) *p)) diff --git a/tools/ompipe.c b/tools/ompipe.c index 52f1c60e..1a9d610f 100644 --- a/tools/ompipe.c +++ b/tools/ompipe.c @@ -71,6 +71,7 @@ DEFobjCurrIf(errmsg) typedef struct _instanceData { uchar f_fname[MAXFNAME];/* pipe or template name (display only) */ short fd; /* pipe descriptor for (current) pipe */ + sbool bHadError; /* did we already have/report an error on this pipe? */ } instanceData; @@ -100,6 +101,17 @@ preparePipe(instanceData *pData) { DEFiRet; pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); + if(pData->fd < 0 ) { + pData->fd = -1; + if(!pData->bHadError) { + char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output pipe '%s': %s", + pData->f_fname, errStr); + pData->bHadError = 1; + } + DBGPRINTF("Error opening log pipe: %s\n", pData->f_fname); + } RETiRet; } @@ -149,6 +161,7 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->fd = -1; + pData->bHadError = 0; ENDcreateInstance @@ -203,11 +216,6 @@ CODESTARTparseSelectorAct */ preparePipe(pData); - if(pData->fd < 0 ) { - pData->fd = -1; - DBGPRINTF("Error opening log pipe: %s\n", pData->f_fname); - errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could not open output pipe '%s'", pData->f_fname); - } CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c index a8ad568b..9130e403 100644 --- a/tools/omusrmsg.c +++ b/tools/omusrmsg.c @@ -269,6 +269,7 @@ ENDdoAction BEGINparseSelectorAct uchar *q; int i; + int bHadWarning = 0; CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1) /* User names must begin with a gnu e-regex: @@ -277,9 +278,18 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) */ if(!strncmp((char*) p, ":omusrmsg:", sizeof(":omusrmsg:") - 1)) { p += sizeof(":omusrmsg:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ - } else if(!*p || !((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') - || (*p >= '0' && *p <= '9') || *p == '_' || *p == '.' || *p == '*')) + } else { + if(!*p || !((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') + || (*p >= '0' && *p <= '9') || *p == '_' || *p == '.' || *p == '*')) { ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } else { + errmsg.LogError(0, RS_RET_OUTDATED_STMT, + "action '%s' treated as ':omusrmsg:%s' - please " + "change syntax, '%s' will not be supported in the future", + p, p, p); + bHadWarning = 1; + } + } CHKiRet(createInstance(&pData)); @@ -313,6 +323,8 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) != RS_RET_OK) goto finalize_it; } + if(iRet == RS_RET_OK && bHadWarning) + iRet = RS_RET_OK_WARN; CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct diff --git a/tools/syslogd.c b/tools/syslogd.c index 0b7bbc96..9553ad08 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -334,6 +334,8 @@ static uchar template_WallFmt[] = "\"\r\n\7Message from syslogd@%HOSTNAME% at %t static uchar template_StdUsrMsgFmt[] = "\" %syslogtag%%msg%\n\r\""; static uchar template_StdDBFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-mysql%', '%timegenerated:::date-mysql%', %iut%, '%syslogtag%')\",SQL"; static uchar template_StdPgSQLFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%')\",STDSQL"; +static uchar template_SysklogdFileFormat[] = "\"%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%\n\""; +static uchar template_StdJSONFmt[] = "\"{\\\"message\\\":\\\"%msg%\\\",\\\"fromhost\\\":\\\"%HOSTNAME%\\\",\\\"facility\\\":\\\"%syslogfacility-text%\\\",\\\"priority\\\":\\\"%syslogpriority-text%\\\",\\\"timereported\\\":\\\"%timereported:::date-rfc3339%\\\",\\\"timegenerated\\\":\\\"%timegenerated:::date-rfc3339%\\\"}\",JSON"; static uchar template_spoofadr[] = "\"%fromhost-ip%\""; /* end templates */ @@ -459,8 +461,15 @@ void untty(void) #else { int i; + pid_t pid; if(!Debug) { + pid = getpid(); + if (setpgid(pid, pid) < 0) { + perror("setpgid"); + exit(1); + } + i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) @@ -2171,8 +2180,12 @@ static rsRetVal mainThread() tplAddLine(" StdUsrMsgFmt", &pTmp); pTmp = template_StdDBFmt; tplAddLine(" StdDBFmt", &pTmp); + pTmp = template_SysklogdFileFormat; + tplAddLine("RSYSLOG_SysklogdFileFormat", &pTmp); pTmp = template_StdPgSQLFmt; tplAddLine(" StdPgSQLFmt", &pTmp); + pTmp = template_StdJSONFmt; + tplAddLine(" StdJSONFmt", &pTmp); pTmp = template_spoofadr; tplLastStaticInit(tplAddLine("RSYSLOG_omudpspoofDfltSourceTpl", &pTmp)); |