diff options
76 files changed, 2054 insertions, 152 deletions
@@ -1,18 +1,31 @@ +--------------------------------------------------------------------------- +Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? - new feature: new output plugin omprog, which permits to start program and feed it (via its stdin) with syslog messages. If the program terminates, it is restarted. -- bugfix: potential abort with DA queue after high watermark is reached - There exists a race condition that can lead to a segfault. Thanks - go to vbernetr, who performed the analysis and provided patch, which - I only tweaked a very little bit. ---------------------------------------------------------------------------- -Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +- improved internal handling of RainerScript functions, building the + necessary plumbing to support more functions with decent runtime + performance. This is also necessary towards the long-term goal + of loadable library modules. +- added new RainerScript function "tolower" +- improved testbench + * added tests for tcp-based reception + * added tcp-load test (1000 connections, 20,000 messages) +- added $MaxOpenFiles configuration directive +- bugfix: solved potential memory leak in msg processing, could manifest + itself in imtcp +--------------------------------------------------------------------------- +Version 4.1.7 [BETA] (rgerhards), 2009-04-?? +- bugfix: $InputTCPMaxSessions config directive was accepted, but not + honored. This resulted in a fixed upper limit of 200 connections. +--------------------------------------------------------------------------- +Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 +- added new "csv" property replacer options to enable simple creation + of CSV-formatted outputs (format from RFC4180 is used) - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet supported - but we now have a good foundation to do that later on. - NOTE: nested function calls are not yet supported due to a design - issue with the function call VM instruction set design. - implemented the strlen() RainerScript function - added a template output module - added -T rsyslogd command line option, enables to specify a directory @@ -35,6 +48,18 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? the full high availability features of rsyslog's engine - bugfix: fixed some segaults on Solaris, where vsprintf() does not check for NULL pointers +- improved performance of regexp-based filters + Thanks to Arnaud Cornet for providing the idea and initial patch. +- added a new way how output plugins may be passed parameters. This is + more effcient for some outputs. They new can receive fields not only + as a single string but rather in an array where each string is seperated. +- added (some) developer documentation for output plugin interface +- bugfix: potential abort with DA queue after high watermark is reached + There exists a race condition that can lead to a segfault. Thanks + go to vbernetr, who performed the analysis and provided patch, which + I only tweaked a very little bit. +- bugfix: imtcp did incorrectly parse hostname/tag + Thanks to Luis Fernando Muñoz Mejías for the patch. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages @@ -130,8 +155,13 @@ version before switching to this one. - bugfix: memory leak in ompgsql Thanks to Ken for providing the patch --------------------------------------------------------------------------- -Version 3.21.11 [BETA] (rgerhards), 2009-0?-?? +Version 3.22.0 [v3-stable] (rgerhards), 2009-04-?? +- bugfix: $InputTCPMaxSessions config directive was accepted, but not + honored. This resulted in a fixed upper limit of 200 connections. +--------------------------------------------------------------------------- +Version 3.21.11 [BETA] (rgerhards), 2009-04-03 - build system improvements contributed by Michael Biebl - thx! +- all patches from 3.20.5 incorporated (see it's ChangeLog entry) --------------------------------------------------------------------------- Version 3.21.10 [BETA] (rgerhards), 2009-02-02 - bugfix: inconsistent use of mutex/atomic operations could cause segfault @@ -283,7 +313,20 @@ Version 3.21.0 [DEVEL] (rgerhards), 2008-07-18 - imported all changes from 3.18.1 until today (some quite important, see below) --------------------------------------------------------------------------- -Version 3.20.5 [v3-stable] (rgerhards), 2009-0?-?? +Version 3.20.6 [v3-stable] (rgerhards), 2009-04-?? +- bugfix: $InputTCPMaxSessions config directive was accepted, but not + honored. This resulted in a fixed upper limit of 200 connections. +--------------------------------------------------------------------------- +Version 3.20.5 [v3-stable] (rgerhards), 2009-04-02 +- bugfix: potential abort with DA queue after high watermark is reached + There exists a race condition that can lead to a segfault. Thanks + go to vbernetr, who performed the analysis and provided patch, which + I only tweaked a very little bit. +- fixed bugs in RainerScript: + o when converting a number and a string to a common type, both were + actually converted to the other variable's type. + o the value of rsCStrConvertToNumber() was miscalculated. + Thanks to varmojfekoj for the patch - fixed a bug in configure.ac which resulted in problems with environment detection - thanks to Michael Biebl for the patch - fixed a potential segfault problem in gssapi code @@ -1209,6 +1252,9 @@ Version 2.0.7 V2-STABLE (rgerhards), 2008-??-?? The actual code change is heavily based on William's patch. - bugfix: memory leak in ompgsql Thanks to Ken for providing the patch +- bugfix: potential memory leak in msg.c + This one did not surface yet and the issue was actually found due to + a problem in v4 - but better fix it here, too --------------------------------------------------------------------------- Version 2.0.6 V2-STABLE (rgerhards), 2008-08-07 - bugfix: memory leaks in rsyslogd, primarily in singlethread mode diff --git a/Makefile.am b/Makefile.am index 02444d6f..a5cf879c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -116,10 +116,15 @@ if ENABLE_RFC3195 SUBDIRS += plugins/im3195 endif +if ENABLE_ORACLE +SUBDIRS += plugins/omoracle +endif + # tests are added as last element, because tests may need different # modules that need to be generated first SUBDIRS += tests + # make sure "make distcheck" tries to build all modules. This means that # a developer must always have an environment where every supporting library # is available. If that is not the case, the respective configure option may @@ -425,6 +425,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) DEFiRet; int iRetries; int i; + int iArr; int iSleepPeriod; int bCallAction; int iCancelStateSave; @@ -439,7 +440,15 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + switch(pAction->eParamPassing) { + case ACT_STRING_PASSING: + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + break; + case ACT_ARRAY_PASSING: + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(ppMsgs[i]))); + break; + default:assert(0); /* software bug if this happens! */ + } } iRetries = 0; /* We now must guard the output module against execution by multiple threads. The @@ -495,7 +504,19 @@ finalize_it: /* cleanup */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { if(ppMsgs[i] != NULL) { - d_free(ppMsgs[i]); + switch(pAction->eParamPassing) { + case ACT_ARRAY_PASSING: + iArr = 0; + while(((char **)ppMsgs[i])[iArr] != NULL) + d_free(((char **)ppMsgs[i])[iArr++]); + d_free(ppMsgs[i]); + break; + case ACT_STRING_PASSING: + d_free(ppMsgs[i]); + break; + default: + assert(0); + } } } d_free(ppMsgs); @@ -905,6 +926,13 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques ABORT_FINALIZE(RS_RET_RQD_TPLOPT_MISSING); } + /* set parameter-passing mode */ + if(iTplOpts & OMSR_TPL_AS_ARRAY) { + pAction->eParamPassing = ACT_ARRAY_PASSING; + } else { + pAction->eParamPassing = ACT_STRING_PASSING; + } + dbgprintf("template: '%s' assigned\n", pTplName); } @@ -61,6 +61,8 @@ struct action_s { short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ + enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1 } + eParamPassing; /* mode of parameter passing to action */ int iNumTpls; /* number of array entries for template element below */ struct template **ppTpl;/* array of template to use - strings must be passed to doAction * in this order. */ diff --git a/configure.ac b/configure.ac index e9e68cb4..c7f1d532 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.5],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.6],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) @@ -452,7 +452,39 @@ AM_CONDITIONAL(ENABLE_PGSQL, test x$enable_pgsql = xyes) AC_SUBST(PGSQL_CFLAGS) AC_SUBST(PGSQL_LIBS) - +# oracle (OCI) support +AC_ARG_ENABLE(oracle, + [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_oracle="yes" ;; + no) enable_oracle="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-oracle) ;; + esac], + [enable_oracle=no] +) +if test "x$enable_oracle" = "xyes"; then + AC_CHECK_PROG( + [HAVE_ORACLE_CONFIG], + [oracle-instantclient-config], + [yes],,, + ) + if test "x${HAVE_ORACLE_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([oracle-instantclient-config not found in PATH]) + fi + AC_CHECK_LIB( + [occi], + [OCIEnvCreate], + [ORACLE_CFLAGS="`oracle-instantclient-config --cflags`" + ORACLE_LIBS="`oracle-instantclient-config --libs`" + ], + [AC_MSG_FAILURE([Oracle (OCI) libraray is missing])], + [`oracle-instantclient-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ORACLE, test x$enable_oracle = xyes) +AC_SUBST(ORACLE_CFLAGS) +AC_SUBST(ORACLE_LIBS) + # libdbi support AC_ARG_ENABLE(libdbi, [AS_HELP_STRING([--enable-libdbi],[Enable libdbi database support @<:@default=no@:>@])], @@ -747,6 +779,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omlibdbi/Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ + plugins/omoracle/Makefile \ tests/Makefile]) AC_OUTPUT @@ -760,6 +793,7 @@ echo "Zlib compression support enabled: $enable_zlib" echo "MySql support enabled: $enable_mysql" echo "libdbi support enabled: $enable_libdbi" echo "PostgreSQL support enabled: $enable_pgsql" +echo "Oracle (OCI) support enabled: $enable_oracle" echo "SNMP support enabled: $enable_snmp" echo "Mail support enabled: $enable_mail" echo "RELP support enabled: $enable_relp" diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html new file mode 100644 index 00000000..5bfc974c --- /dev/null +++ b/doc/dev_oplugins.html @@ -0,0 +1,178 @@ +<html> +<head> +<title>writing rsyslog output plugins (developer's guide)</title> +</head> +<body> +<h1>Writing Rsyslog Output Plugins</h1> +<p>This page is the begin of some developer documentation for writing output +plugins. Doing so is quite easy (and that was a design goal), but there currently +is only sparse documentation on the process available. I was tempted NOT to +write this guide here because I know I will most probably not be able to +write a complete guide. +<p>However, I finally concluded that it may be better to have same information +and pointers than to have nothing. +<h2>Getting Started and Samples</h2> +<p>The best to get started with rsyslog plugin development is by looking at +existing plugins. All that start with "om" are <b>o</b>utput <b>m</b>odules. That +means they are primarily thought of being message sinks. In theory, however, output +plugins may aggergate other functionality, too. Nobody has taken this route so far +so if you would like to do that, it is highly suggested to post your plan on the +rsyslog mailing list, first (so that we can offer advise). +<p>The rsyslog distribution tarball contains two plugins that are extremely well +targeted for getting started: +<ul> +<li>omtemplate +<li>omstdout +</ul> +Plugin omtemplate was specifically created to provide a copy template for new output +plugins. It is bare of real functionality but has ample comments. Even if you decide +to start from another plugin (or even from scratch), be sure to read omtemplate source +and comments first. The omstdout is primarily a testing aide, but offers support for +the two different parameter-passing conventions plugins can use (plus the way to +differentiate between the two). It also is not bare of functionaly, only mostly +bare of it ;). But you can actually execute it and play with it. +<p>In any case, you should also read the comments in ./runtime/module-template.h. +Output plugins are build based on a large set of code-generating macros. These +macros handle most of the plumbing needed by the interface. As long as no +special callback to rsyslog is needed (it typically is not), an output plugin does +not really need to be aware that it is executed by rsyslog. As a plug-in programmer, +you can (in most cases) "code as usual". However, all macros and entry points need to be +provided and thus reading the code comments in the files mentioned is highly suggested. +<p>In short, the best idea is to start with a template. Let's assume you start by +copying omtemplate. Then, the basic steps you need to do are: +<ul> +<li>cp ./plugins/omtemplate ./plugins/your-plugin +<li>mv cd ./plugins/your-plugin +<li>vi Makefile.am, adjust to your-plugin +<li>mv omtemplate.c your-plugin.c +<li>cd ../.. +<li>vi Makefile.am configure.ac +<br>serach for omtemplate, copy and modify (follow comments) +</ul> +<p>Basically, this is all you need to do ... Well, except, of course, coding +your plugin ;). For testing, you need rsyslog's debugging support. Some useful +information is given in "<a href="troubleshoot.html">troubleshooting rsyslog</a> +from the doc set. +<h2>Special Topics</h2> +<h3>Threading</h3> +<p>Rsyslog uses massive parallel processing and multithreading. However, a plugin's entry +points are guaranteed to be never called concurrently <b>for the same action</b>. +That means your plugin must be able to be called concurrently by two or more +threads, but you can be sure that for the same instance no concurrent calls +happen. This is guaranteed by the interface specification and the rsyslog core +guards against multiple concurrent calls. An instance, in simple words, is one +that shares a single instanceData structure. +<p>So as long as you do not mess around with global data, you do not need +to think about multithreading (and can apply a purely sequential programming +methodology). +<p>Please note that duringt the configuraton parsing stage of execution, access to +global variables for the configuration system is safe. In that stage, the core will +only call sequentially into the plugin. +<h3>Getting Message Data</h3> +<p>The doAction() entry point of your plugin is provided with messages to be processed. +It will only be activated after filtering and all other conditions, so you do not need +to apply any other conditional but can simply process the message. +<p>Note that you do NOT receive the full internal representation of the message +object. There are various (including historical) reasons for this and, among +others, this is a design decision based on security. +<p>Your plugin will only receive what the end user has configured in a $template +statement. However, starting with 4.1.6, there are two ways of receiving the +template content. The default mode, and in most cases sufficient and optimal, +is to receive a single string with the expanded template. As I said, this is usually +optimal, think about writing things to files, emailing content or forwarding it. +<p>The important philosophy is that a plugin should <b>never</b> reformat any +of such strings - that would either remove the user's ability to fully control +message formats or it would lead to duplicating code that is already present in the +core. If you need some formatting that is not yet present in the core, suggest it +to the rsyslog project, best done by sending a patch ;), and we will try hard to +get it into the core (so far, we could accept all such suggestions - no promise, though). +<p>If a single string seems not suitable for your application, the plugin can also +request access to the template components. The typical use case seems to be databases, where +you would like to access properties via specific fields. With that mode, you receive a +char ** array, where each array element points to one field from the template (from +left to right). Fields start at arrray index 0 and a NULL pointer means you have +reached the end of the array (the typical Unix "poor man's linked list in an array" +design). Note, however, that each of the individual components is a string. It is +not a date stamp, number or whatever, but a string. This is because rsyslog processes +strings (from a high-level design look at it) and so this is the natural data type. +Feel free to convert to whatever you need, but keep in mind that malformed packets +may have lead to field contents you'd never expected... +<p>If you like to use the array-based parameter passing method, think that it +is only available in rsyslog 4.1.6 and above. If you can accept that your plugin +will not be working with previous versions, you do not need to handle pre 4.1.6 cases. +However, it would be "nice" if you shut down yourself in these cases - otherwise the +older rsyslog core engine will pass you a string where you expect the array of pointers, +what most probably results in a segfault. To check whether or not the core supports the +functionality, you can use this code sequence: +<pre> +<code> +BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bArrayPassingSupported; /* does core support template passing as an array? */ +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + /* check if the rsyslog core supports parameter passing code */ + bArrayPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + /* found entry point, so let's see if core supports array passing */ + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_ARRAY) + bArrayPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */ + } + DBGPRINTF("omstdout: array-passing is %ssupported by rsyslog core.\n", bArrayPassingSupported ? "" : "not "); + + if(!bArrayPassingSupported) { + DBGPRINTF("rsyslog core too old, shutting down this plug-in\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + +</code> +</pre> +<p>The code first checks if the core supports the OMSRgetSupportedTplOpts() API (which is +also not present in all versions!) and, if so, queries the core if the OMSR_TPL_AS_ARRAY mode +is supported. If either does not exits, the core is too old for this functionality. The sample +snippet above then shuts down, but a plugin may instead just do things different. In +omstdout, you can see how a plugin may deal with the situation. +<p><b>In any case, it is recommended that at least a graceful shutdown is made and the +array-passing capability not blindly be used.</b> In such cases, we can not guard the +plugin from segfaulting and if the plugin (as currently always) is run within +rsyslog's process space, that results in a segfault for rsyslog. So do not do this. +<h3>Batching of Messages</h3> +<p>With the current plugin interface, each message is passed via a separate call to the plugin. +This is annoying and costs performance in some uses cases (primarily for database outputs). +However, that's the way it (currently) is, no easy way around it. There are some ideas +to implement batching capabilities inside the rsyslog core, but without that the only +resort is to do it inside your plugin yourself. You are not prohibited from doing so. +There are some consequences, though: most importantly, the rsyslog core is no longer +intersted in messages that it passed to a plugin. As such, it will not try to make sure +the message is not lost before it was ultimately processed (because rsyslog, due to +doAction() returning successfully, thinks the message *was* ultimately processed). +<p>When the rsyslog core receives batching capabilities, this will be implemented in +a way that is fully compatible to the existing plugin interface. While we have not yet +thought about the implementation, that will probably mean that some new interfaces +or options be used to turn on batching capabilities. +<h3>Licensing</h3> +<p>From the rsyslog point of view, plugins constitute separate projects. As such, +we think plugins are not required to be compatible with GPLv3. However, this is +no legal advise. If you intend to release something under a non-GPLV3 compatible license +it is probably best to consult with your lawyer. +<p>Most importantly, and this is definite, the rsyslog team does not expect +or require you to contribute your plugin to the rsyslog project (but of course +we are happy if you do). +<h2>Copyright</h2> +<p>Copyright (c) 2009 <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> +and <a href="http://www.adiscon.com/en/">Adiscon</a>.</p> +<p>Permission is granted to copy, distribute and/or modify this document under +the terms of the GNU Free Documentation License, Version 1.2 or any later +version published by the Free Software Foundation; with no Invariant Sections, +no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be +viewed at <a href="http://www.gnu.org/copyleft/fdl.html"> +http://www.gnu.org/copyleft/fdl.html</a>.</p> +</body> +</html> diff --git a/doc/features.html b/doc/features.html index 336b31cc..626ff65d 100644 --- a/doc/features.html +++ b/doc/features.html @@ -95,6 +95,9 @@ via custom plugins</li> <li> an easy-to-write to plugin interface</li> <li> ability to send SNMP trap messages</li> <li> ability to filter out messages based on sequence of arrival</li> +<li>support for comma-seperated-values (CSV) output generation +(via the "csv" property replace option). The +CSV format supported is that from RFC 4180.</li> <li>support for arbitrary complex boolean, string and arithmetic expressions in message filters</li> </ul> diff --git a/doc/manual.html b/doc/manual.html index 99a90594..51271701 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 4.1.5 (devel branch) of rsyslog.</b> +<p><b>This documentation is for version 4.1.6 (devel branch) of rsyslog.</b> Visit the <i> <a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might @@ -64,7 +64,11 @@ the syslog priority (severity and facility) to the log file</a></li> syslog sender over NAT</a> (online only)</li> <li><a href="gssapi.html">an overview and howto of rsyslog gssapi support</a></li> <li><a href="debug.html">debug support in rsyslog</a></li> -<li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> +<li>Developer Documentation + <ul> + <li><a href="dev_oplugins.html">writing rsyslog output plugins</a></li> + <li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> + </ul></li> </ul> <p>Our <a href="history.html">rsyslog history</a> page is for you if you would like to learn a little more @@ -83,6 +87,7 @@ wiki</a>, a community resource which includes <a href="http://wiki.rsyslog. online documentation (most current version only)</a></li> <li><a href="http://kb.monitorware.com/rsyslog-f40.html">rsyslog discussion forum - use this for technical support</a></li> +<li><a href="http://www.rsyslog.com/Topic8.phtml">rsyslog video tutorials</a></li> <li><a href="http://www.rsyslog.com/Topic4.phtml">rsyslog change log</a></li> <li><a href="http://www.rsyslog.com/Topic3.phtml">rsyslog FAQ</a></li> <li><a href="http://www.monitorware.com/en/syslog-enabled-products/">syslog device configuration guide</a> (off-site)</li> diff --git a/doc/property_replacer.html b/doc/property_replacer.html index baf053f2..a6e9b518 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -314,6 +314,18 @@ case-insensitive. Currently, the following options are defined: <td>convert property text to uppercase only</td> </tr> <tr> +<td valign="top"><b>csv</b></td> +<td>formats the resulting field (after all modifications) in CSV format +as specified in <a href="http://www.ietf.org/rfc/rfc4180.txt">RFC 4180</a>. +Rsyslog will always use double quotes. Note that in order to have full CSV-formatted +text, you need to define a proper template. An example is this one: +<br>$template csvline,"%syslogtag:::csv%,%msg:::csv%" +<br>Most importantly, you need to provide the commas between the fields +inside the template. +<br><i>This feature was introduced in rsyslog 4.1.6.</i> +</td> +</tr> +<tr> <td><b>drop-last-lf</b></td> <td>The last LF in the message (if any), is dropped. Especially useful for PIX.</td> @@ -411,7 +423,7 @@ syntax</a>, this is where you actually use the property replacer.</li> [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> diff --git a/doc/rsconf1_maxopenfiles.html b/doc/rsconf1_maxopenfiles.html new file mode 100644 index 00000000..b6c9cc0e --- /dev/null +++ b/doc/rsconf1_maxopenfiles.html @@ -0,0 +1,35 @@ +<html> +<head> +<title>$MaxOpenFiles - rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">[rsyslog configuration directive overview]</a> + +<h2>$MaxOpenFiles</h2> +<p><b>Available Since:</b> 4.3.0</p> +<p><b>Type:</b> global configuration directive</p> +<p><b>Default:</b> <i>operating system default</i></p> +<p><b>Description:</b></p> +<p>Set the maximum number of files that the rsyslog process can have open at any given +time. Note that this includes open tcp sockets, so this setting is the upper limit for +the number of open TCP connections as well. If you expect a large nubmer of concurrent +connections, it is suggested that the number is set to the max number connected plus 1000. +Please note that each dynafile also requires up to 100 open file handles. +<p>The setting is similar to running "ulimit -n number-of-files". +<p>Please note that depending on permissions and operating system configuration, the +setrlimit() request issued by rsyslog may fail, in which case the previous limit is kept +in effect. Rsyslog will emit a warning message in this case. +<p><b>Sample:</b></p> +<p><code><b>$MaxOpenFiles 2000</b></code></p> +<p><b>Bugs:</b></p> +<p>For some reason, this settings seems not to work on all platforms. If you experience +problems, please let us know so that we can (hopefully) narrow down the issue. +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 852d95b5..6990c6bd 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -26,7 +26,7 @@ Lines can be continued by specifying a backslash ("\") as the last character of the line. There is a hard-coded maximum line length of 4K. If you need lines larger than that, you need to change compile-time settings inside rsyslog and recompile. -<h2><a href="rsyslog_conf_global.html">Global Directives</a></h2> +<h2><a href="rsyslog_conf_global.html">Configuration Directives</a></h2> <h2>Basic Structure</h2> <p>Rsyslog supports standard sysklogd's configuration file format and extends it. So in general, you can take a "normal" syslog.conf and @@ -74,9 +74,9 @@ such features is available in rsyslogd, only.</p> [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008,2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> > diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d011bd2b..3e33f0da 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -1,14 +1,14 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html><head><title>Global Directives - rsyslog.conf</title></head> +<html><head><title>Configuration Directives - rsyslog.conf</title></head> <body> <p>This is a part of the rsyslog.conf documentation.</p> <a href="rsyslog_conf.html">back</a> -<h2>Global Directives</h2> -<p>All global directives need to be specified on a line by their -own and must start with a dollar-sign. Here is a list in alphabetical -order. Follow links for a description.</p> -<p>Please note that not all directives here are actually global. Some affect -only the next action. This documentation will be changed soon. +<h2>Configuration Directives</h2> +<p>All configuration directives need to be specified on a line by their +own and must start with a dollar-sign. Note that those starting with +the word "Action" modify the next action and should be specified +in front of it. +<p>Here is a list in alphabetical order. Follow links for a description.</p> <p>Not all directives have an in-depth description right now. Default values for them are in bold. A more in-depth description will appear as implementation progresses. @@ -180,6 +180,7 @@ instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restri <br>Note that 2k, the current default, is the smallest size that must be supported in order to be compliant to the upcoming new syslog RFC series. </li> +<li><a href="rsconf1_maxopenfiles.html">$MaxOpenFiles</a></li> <li><a href="rsconf1_moddir.html">$ModDir</a></li> <li><a href="rsconf1_modload.html">$ModLoad</a></li> <li><b>$RepeatedMsgContainsOriginalMsg</b> [on/<b>off</b>] - "last message repeated n times" messages, if generated, @@ -214,7 +215,6 @@ the value, the less precise the timestamp. <li><a href="droppriv.html">$PrivDropToGroupID</a></li> <li><a href="droppriv.html">$PrivDropToUser</a></li> <li><a href="droppriv.html">$PrivDropToUserID</a></li> -</ul> <li><a href="rsconf1_umask.html">$UMASK</a></li> </ul> <p><b>Where <size_nbr> is specified above,</b> @@ -235,7 +235,7 @@ point of view, "1,,0.0.,.,0" also has the value 1000. </p> <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 890a55c8..a281d9e7 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -8,10 +8,9 @@ number of modules. Here is the entry point to their documentation and what they do (list is currently not complete)</p> <ul> -<li><a href="omsnmp.html">omsnmp</a> - SNMP -trap output module</li> -<li><a href="omrelp.html">omrelp</a> - RELP -output module</li> +<li><a href="omsnmp.html">omsnmp</a> - SNMP trap output module</li> +<li><a href="omstdout.html">omtdout</a> - stdout output module (mainly a test tool)</li> +<li><a href="omrelp.html">omrelp</a> - RELP output module</li> <li>omgssapi - output module for GSS-enabled syslog</li> <li><a href="ommysql.html">ommysql</a> - output module for MySQL</li> <li>ompgsql - output module for PostgreSQL</li> diff --git a/doc/status.html b/doc/status.html index 59fd0809..dae94884 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,19 +2,19 @@ <html><head><title>rsyslog status page</title></head> <body> <h2>rsyslog status page</h2> -<p>This page reflects the status as of 2009-02-02.</p> +<p>This page reflects the status as of 2009-04-03.</p> <h2>Current Releases</h2> <p><b>development:</b> 4.1.5 [2009-03-11] - <a href="http://www.rsyslog.com/Article349.phtml">change log</a> - <a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-150.phtml">download</a> -<br><b>beta:</b> 3.21.10 [2009-02-02] - -<a href="http://www.rsyslog.com/Article344.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-148.phtml">download</a></p> +<br><b>beta:</b> 3.21.11 [2009-04-03] - +<a href="http://www.rsyslog.com/Article358.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-152.phtml">download</a></p> -<p><b>v3 stable:</b> 3.20.3 [2009-01-19] - <a href="http://www.rsyslog.com/Article339.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-146.phtml">download</a> +<p><b>v3 stable:</b> 3.20.3 [2009-04-02] - <a href="http://www.rsyslog.com/Article356.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-151.phtml">download</a> <br><b>v2 stable:</b> 2.0.6 [2008-08-07] - <a href="http://www.rsyslog.com/Article266.phtml">change log</a> - <a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-125.phtml">download</a> diff --git a/plugins/im3195/im3195.c b/plugins/im3195/im3195.c index 1c2502fe..106da2c8 100644 --- a/plugins/im3195/im3195.c +++ b/plugins/im3195/im3195.c @@ -47,6 +47,7 @@ #include "liblogging/syslogmessage.h" #include "module-template.h" #include "cfsysline.h" +#include "msg.h" #include "errmsg.h" MODULE_TYPE_INPUT @@ -83,7 +84,7 @@ void OnReceive(srAPIObj __attribute__((unused)) *pMyAPI, srSLMGObj* pSLMG) srSLMGGetRawMSG(pSLMG, &pszRawMsg); parseAndSubmitMessage(fromHost, fromHostIP, pszRawMsg, strlen((char*)pszRawMsg), - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_FULL_DELAY, (uchar*)"im3195"); + PARSE_HOSTNAME, eFLOWCTL_FULL_DELAY, (uchar*)"im3195", NULL, 0); } diff --git a/plugins/imklog/linux.c b/plugins/imklog/linux.c index 198b7c0e..0dd4320d 100644 --- a/plugins/imklog/linux.c +++ b/plugins/imklog/linux.c @@ -144,7 +144,7 @@ static enum LOGSRC GetKernelLogSrc(void) return(kernel); } - if ( (kmsg = open(_PATH_KLOG, O_RDONLY)) < 0 ) + if ( (kmsg = open(_PATH_KLOG, O_RDONLY|O_CLOEXEC)) < 0 ) { imklogLogIntMsg(LOG_ERR, "imklog: Cannot open proc file system, %d.\n", errno); ksyslog(7, NULL, 0); /* TODO: check this, implement more */ diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 19138d94..5a8a62f6 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -162,6 +162,7 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa if(pOurTcpsrv == NULL) { CHKiRet(tcpsrv.Construct(&pOurTcpsrv)); + CHKiRet(tcpsrv.SetSessMax(pOurTcpsrv, iTCPSessMax)); CHKiRet(tcpsrv.SetCBIsPermittedHost(pOurTcpsrv, isPermittedHost)); CHKiRet(tcpsrv.SetCBRcvData(pOurTcpsrv, doRcvData)); CHKiRet(tcpsrv.SetCBOpenLstnSocks(pOurTcpsrv, doOpenLstnSocks)); diff --git a/plugins/omoracle/Makefile.am b/plugins/omoracle/Makefile.am new file mode 100644 index 00000000..11257fb2 --- /dev/null +++ b/plugins/omoracle/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omoracle.la + +omoracle_la_SOURCES = omoracle.c omoracle.h +omoracle_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(ORACLE_CFLAGS) +omoracle_la_LDFLAGS = -module -avoid-version +omoracle_la_LIBADD = $(ORACLE_LIBS) + +#EXTRA_DIST = diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c new file mode 100644 index 00000000..ea910d3a --- /dev/null +++ b/plugins/omoracle/omoracle.c @@ -0,0 +1,334 @@ +/** omoracle.c + + This is an output module feeding directly to an Oracle + database. It uses Oracle Call Interface, a propietary module + provided by Oracle. + + Selector lines to be used are of this form: + + :omoracle:;TemplateName + + The module gets its configuration via rsyslog $... directives, + namely: + + $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db + name as specified by tnsnames.ora) + + All fields are mandatory. The dbstring can be an Oracle easystring + or a DB name, as present in the tnsnames.ora file. + + Author: Luis Fernando Muñoz Mejías + <Luis.Fernando.Munoz.Mejias@cern.ch> + + This file is part of rsyslog. +*/ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <oci.h> +#include <errno.h> +#include <stdarg.h> +#include <signal.h> +#include <time.h> +#include <assert.h> +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" +#include "omoracle.h" + +MODULE_TYPE_OUTPUT + +/** */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +typedef struct _instanceData { + /* Environment handler, the base for any OCI work. */ + OCIEnv* environment; + /* Session handler, the actual DB connection object. */ + OCISession* session; + /* Error handler for OCI calls. */ + OCIError* error; + /* Prepared statement. */ + OCIStmt* statement; + /* Service handler. */ + OCISvcCtx* service; + /* Credentials object for the connection. */ + OCIAuthInfo* authinfo; + /* Binding parameters, currently unused */ + OCIBind* binding; + /* Connection string, kept here for possible retries. */ + char* connection; +} instanceData; + +/** Database name, to be filled by the $OmoracleDB directive */ +static char* db_name; +/** Database user name, to be filled by the $OmoracleDBUser + * directive */ +static char* db_user; +/** Database password, to be filled by the $OmoracleDBPassword */ +static char* db_password; + +/** Generic function for handling errors from OCI. + + It will be called only inside CHECKERR and CHECKENV macros. + + Arguments: handle The error or environment handle to check. + htype: OCI_HTYPE_* constant, usually OCI_HTYPE_ERROR or + OCI_HTYPE_ENV + status: status code to check, usually the return value of an OCI + function. + + Returns OCI_SUCCESS on success, OCI_ERROR otherwise. +*/ +static int oci_errors(void* handle, ub4 htype, sword status) +{ + sb4 errcode; + unsigned char buf[MAX_BUFSIZE]; + + switch (status) { + case OCI_SUCCESS: + return OCI_SUCCESS; + break; + case OCI_SUCCESS_WITH_INFO: + errmsg.LogError(0, NO_ERRCODE, "OCI SUCCESS - With info\n"); + break; + case OCI_NEED_DATA: + errmsg.LogError(0, NO_ERRCODE, "OCI NEEDS MORE DATA\n"); + break; + case OCI_ERROR: + dbgprintf ("OCI GENERAL ERROR\n"); + if (handle) { + OCIErrorGet(handle, 1, NULL, &errcode, buf, + sizeof buf, htype); + errmsg.LogError(0, NO_ERRCODE, "Error message: %s", buf); + } else + errmsg.LogError(0, NO_ERRCODE, "NULL handle\n" + "Unable to extract further " + "information"); + break; + case OCI_INVALID_HANDLE: + errmsg.LogError(0, NO_ERRCODE, "OCI INVALID HANDLE\n"); + break; + case OCI_STILL_EXECUTING: + errmsg.LogError(0, NO_ERRCODE, "Still executing...\n"); + break; + case OCI_CONTINUE: + errmsg.LogError(0, NO_ERRCODE, "OCI CONTINUE\n"); + break; + } + return OCI_ERROR; +} + + +/* Resource allocation */ +BEGINcreateInstance +CODESTARTcreateInstance + + ASSERT(pData != NULL); + + CHECKENV(pData->environment, + OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, + NULL, NULL, NULL, NULL, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->error), + OCI_HTYPE_ERROR, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), + OCI_HTYPE_AUTHINFO, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->statement), + OCI_HTYPE_STMT, 0, NULL)); + +finalize_it: +ENDcreateInstance + +/** Close the session and free anything allocated by + createInstance. */ +BEGINfreeInstance +CODESTARTfreeInstance + + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->environment, OCI_HTYPE_ENV); + OCIHandleFree(pData->error, OCI_HTYPE_ERROR); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); + OCIHandleFree(pData->statement, OCI_HTYPE_STMT); + free(pData->connection); + dbgprintf ("omoracle freed all its resources\n"); + RETiRet; + +ENDfreeInstance + +BEGINtryResume +CODESTARTtryResume + /* Here usually only a reconnect is done. The rsyslog core will call + * this entry point from time to time when the action suspended itself. + * Note that the rsyslog core expects that if the plugin suspended itself + * the action was not carried out during that invocation. Thus, rsyslog + * will call the action with *the same* data item again AFTER a resume + * was successful. As such, tryResume should NOT write the failed data + * item. If it needs to for some reason, it must delete the item again, + * otherwise, it will get duplicated. + * This handling inside the rsyslog core is important to be able to + * preserve data over rsyslog restarts. With it, the core can ensure that + * it queues all not-yet-processed messages without the plugin needing + * to take care about that. + * So in essence, it is recommended that just a reconnet is tried, but + * the last action not restarted. Note that it is not a real problem + * (but causes a slight performance degradation) if tryResume returns + * successfully but the next call to doAction() immediately returns + * RS_RET_SUSPENDED. So it is OK to do the actual restart inside doAction(). + * ... of course I don't know why Oracle might need a full restart... + * rgerhards, 2009-03-26 + */ + dbgprintf("Attempting to reconnect to DB server\n"); + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, + pData->connection, + strlen(pData->connection), NULL, 0, + NULL, NULL, NULL, OCI_DEFAULT)); + +finalize_it: +ENDtryResume + +static rsRetVal startSession(instanceData* pData, char* connection, char* user, + char * password) +{ + DEFiRet; + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, user, + strlen(user), OCI_ATTR_USERNAME, pData->error)); + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, password, + strlen(password), OCI_ATTR_PASSWORD, pData->error)); + CHECKERR(pData->error, + OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, connection, + strlen(connection), NULL, 0, NULL, NULL, NULL, + OCI_DEFAULT)); +finalize_it: + if (iRet != RS_RET_OK) + errmsg.LogError(0, NO_ERRCODE, + "Unable to start Oracle session\n"); + RETiRet; +} + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + /* Right now, this module is compatible with nothing. */ + dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); + iRet = RS_RET_INCOMPATIBLE; +ENDisCompatibleWithFeature + +BEGINparseSelectorAct + +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1); + + if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + p += sizeof ":omoracle:" - 1; + + if (*p == '\0' || *p == ',') { + errmsg.LogError(0, NO_ERRCODE, + "Wrong char processing module arguments: %c\n", + *p); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, + OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + CHKiRet(createInstance(&pData)); + CHKmalloc(pData->connection = strdup(db_name)); + CHKiRet(startSession(pData, db_name, db_user, db_password)); + + dbgprintf ("omoracle module got all its resources allocated " + "and connected to the DB\n"); + +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINdoAction +CODESTARTdoAction + dbgprintf("omoracle attempting to execute statement %s\n", *ppString); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, pData->error, *ppString, + strlen(*ppString), OCI_NTV_SYNTAX, + OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, pData->statement, pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); +finalize_it: + dbgprintf ("omoracle %s at executing statement %s\n", + iRet?"did not succeed":"succeeded", *ppString); +/* Clean credentials to avoid leakage in case of core dump. */ +ENDdoAction + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, + void __attribute__((unused)) *pVal) +{ + int n; + DEFiRet; + if(db_user != NULL) + free(db_user); + if(db_name != NULL) + free(db_name); + if (db_password != NULL) { + n = strlen(db_password); + memset(db_password, 0, n); + free(db_password); + } + db_name = db_user = db_password = NULL; + RETiRet; +} + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ + CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, + eCmdHdlrCustomHandler, resetConfigVariables, + NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbuser", 0, + eCmdHdlrGetWord, NULL, &db_user, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbpassword", 0, + eCmdHdlrGetWord, NULL, &db_password, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, + eCmdHdlrGetWord, NULL, &db_name, + STD_LOADABLE_MODULE_ID)); +ENDmodInit diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h new file mode 100644 index 00000000..b0e70917 --- /dev/null +++ b/plugins/omoracle/omoracle.h @@ -0,0 +1,23 @@ +/** Definitions for the Oracle output module. + + This module needs OCI to be installed (on Red Hat-like systems + this is usually the oracle-instantclient-devel RPM). + + Author: Luis Fernando Muñoz Mejías <Luis.Fernando.Munoz.Mejias@cern.ch> +*/ +#ifndef __OMORACLEH__ +#define __OMORACLEH__ + +/** Macros to make error handling easier. */ + +/** Checks for errors on the OCI handling. */ +#define CHECKERR(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ERROR, (status))) + +/** Checks for errors when handling the environment of OCI. */ +#define CHECKENV(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ENV, (status))) + +enum { MAX_BUFSIZE = 2048 }; + +#endif diff --git a/plugins/omoracle/omoracle.te b/plugins/omoracle/omoracle.te new file mode 100644 index 00000000..81eb6cf1 --- /dev/null +++ b/plugins/omoracle/omoracle.te @@ -0,0 +1,13 @@ + +module omoracle 1.0; + +require { + type syslogd_t; + type port_t; + class process { execstack execmem }; + class tcp_socket name_connect; +} + +#============= syslogd_t ============== +allow syslogd_t port_t:tcp_socket name_connect; +allow syslogd_t self:process { execstack execmem }; diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 6e227ba9..181895a4 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -49,7 +49,12 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA +/* config variables */ +static int bUseArrayInterface; /* shall action use array instead of string template interface? */ + + typedef struct _instanceData { + int bUseArrayInterface; /* uses action use array instead of string template interface? */ } instanceData; BEGINcreateInstance @@ -79,12 +84,48 @@ CODESTARTtryResume ENDtryResume BEGINdoAction + char **szParams; + char *toWrite; + int iParamVal; + int iParam; + int iBuf; + char szBuf[65564]; CODESTARTdoAction - write(1, (char*)ppString[0], strlen((char*)ppString[0])); + if(pData->bUseArrayInterface) { + /* if we use array passing, we need to put together a string + * ourselves. At this point, please keep in mind that omstdout is + * primarily a testing aid. Other modules may do different processing + * if they would like to support downlevel versions which do not support + * array-passing, but also use that interface on cores who do... + * So this code here is also more or less an example of how to do that. + * rgerhards, 2009-04-03 + */ + szParams = (char**) (ppString[0]); + /* In array-passing mode, ppString[] contains a NULL-terminated array + * of char *pointers. + */ + iParam = 0; + iBuf = 0; + while(szParams[iParam] != NULL) { + if(iParam > 0) + szBuf[iBuf++] = ','; /* all but first need a delimiter */ + iParamVal = 0; + while(szParams[iParam][iParamVal] != '\0' && iBuf < (int) sizeof(szBuf)) { + szBuf[iBuf++] = szParams[iParam][iParamVal++]; + } + ++iParam; + } + szBuf[iBuf] = '\0'; + toWrite = szBuf; + } else { + toWrite = (char*) ppString[0]; + } + write(1, toWrite, strlen(toWrite)); /* 1 is stdout! */ ENDdoAction BEGINparseSelectorAct + int iTplOpts; CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1) /* first check if this config line is actually for us */ @@ -99,7 +140,9 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) /* check if a non-standard template is to be applied */ if(*(p-1) == ';') --p; - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); + iTplOpts = (bUseArrayInterface == 0) ? 0 : OMSR_TPL_AS_ARRAY; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, iTplOpts, (uchar*) "RSYSLOG_FileFormat")); + pData->bUseArrayInterface = bUseArrayInterface; CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -115,10 +158,45 @@ CODEqueryEtryPt_STD_OMOD_QUERIES ENDqueryEtryPt + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + bUseArrayInterface = 0; + RETiRet; +} + + BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bArrayPassingSupported; /* does core support template passing as an array? */ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + /* check if the rsyslog core supports parameter passing code */ + bArrayPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + /* found entry point, so let's see if core supports array passing */ + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_ARRAY) + bArrayPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */ + } + DBGPRINTF("omstdout: array-passing is %ssupported by rsyslog core.\n", bArrayPassingSupported ? "" : "not "); + + if(bArrayPassingSupported) { + /* enable config comand only if core supports it */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomstdoutarrayinterface", 0, eCmdHdlrBinary, NULL, + &bUseArrayInterface, STD_LOADABLE_MODULE_ID)); + } + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit /* vi:set ai: diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index c4490b48..e1e4a6a4 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -364,7 +364,7 @@ static rsRetVal doGetGID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p /* we set value via a set function */ CHKiRet(pSetHdlr(pVal, pgBuf->gr_gid)); } - dbgprintf("gid %d obtained for group '%s'\n", pgBuf->gr_gid, szName); + dbgprintf("gid %d obtained for group '%s'\n", (int) pgBuf->gr_gid, szName); } skipWhiteSpace(pp); /* skip over any whitespace */ @@ -406,7 +406,7 @@ static rsRetVal doGetUID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p /* we set value via a set function */ CHKiRet(pSetHdlr(pVal, ppwBuf->pw_uid)); } - dbgprintf("uid %d obtained for user '%s'\n", ppwBuf->pw_uid, szName); + dbgprintf("uid %d obtained for user '%s'\n", (int) ppwBuf->pw_uid, szName); } skipWhiteSpace(pp); /* skip over any whitespace */ diff --git a/runtime/datetime.c b/runtime/datetime.c index 676f76d5..deb66eb4 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -479,7 +479,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) if(hour > 1970 && hour < 2100) { /* if so, we assume this actually is a year. This is a format found * e.g. in Cisco devices. - * (if you read this 2100+ trying to fix a bug, congratulate myself + * (if you read this 2100+ trying to fix a bug, congratulate me * to how long the code survived - me no longer ;)) -- rgerhards, 2008-11-18 */ year = hour; diff --git a/runtime/debug.c b/runtime/debug.c index 96004e47..4ee90226 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1343,7 +1343,7 @@ rsRetVal dbgClassInit(void) if(pszAltDbgFileName != NULL) { /* we have a secondary file, so let's open it) */ - if((altdbg = open(pszAltDbgFileName, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY, S_IRUSR|S_IWUSR)) == -1) { + if((altdbg = open(pszAltDbgFileName, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY|O_CLOEXEC, S_IRUSR|S_IWUSR)) == -1) { fprintf(stderr, "alternate debug file could not be opened, ignoring. Error: %s\n", strerror(errno)); } } diff --git a/runtime/expr.c b/runtime/expr.c index 38ed1c68..e449d1c7 100644 --- a/runtime/expr.c +++ b/runtime/expr.c @@ -142,7 +142,9 @@ terminal(expr_t *pThis, ctok_t *tok) * we have all relevant information) */ CHKiRet(ctok_token.UnlinkVar(pToken, &pVar)); - CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_FUNC_CALL, pVar)); /* add to program */ + CHKiRet(var.ConvToString(pVar)); /* make sure we have a string */ + CHKiRet(vmprg.AddCallOperation(pThis->pVmprg, pVar->val.pStr)); /* add to program */ + CHKiRet(var.Destruct(&pVar)); break; case ctok_MSGVAR: dbgoprint((obj_t*) pThis, "MSGVAR\n"); diff --git a/runtime/modules.c b/runtime/modules.c index cef4eac6..9fdb48e7 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -225,6 +225,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) *pEtryPoint = regCfSysLineHdlr; } else if(!strcmp((char*) name, "objGetObjInterface")) { *pEtryPoint = objGetObjInterface; + } else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) { + *pEtryPoint = OMSRgetSupportedTplOpts; } else { *pEtryPoint = NULL; /* to be on the safe side */ ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND); diff --git a/runtime/msg.c b/runtime/msg.c index 9aa2ce84..9d5f3838 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1542,6 +1542,8 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) { assert(pMsg != NULL); assert(pBuf != NULL); + if(pMsg->pszHOSTNAME != NULL) + free(pMsg->pszHOSTNAME); pMsg->iLenHOSTNAME = strlen(pBuf); pMsg->pszHOSTNAME = (uchar*) pBuf; } @@ -1567,7 +1569,7 @@ void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME) if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); else - dbgprintf("Could not allocate memory in MsgSetHOSTNAME()\n"); + DBGPRINTF("Could not allocate memory in MsgSetHOSTNAME()\n"); } @@ -2405,6 +2407,40 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } } + /* finally, we need to check if the property should be formatted in CSV + * format (we use RFC 4180, and always use double quotes). As of this writing, + * this should be the last action carried out on the property, but in the + * future there may be reasons to change that. -- rgerhards, 2009-04-02 + */ + if(pTpe->data.field.options.bCSV) { + /* we need to obtain a private copy, as we need to at least add the double quotes */ + int iBufLen = strlen(pRes); + char *pBStart; + char *pDst; + char *pSrc; + /* the malloc may be optimized, we currently use the worst case... */ + pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(char)); + if(pDst == NULL) { + if(*pbMustBeFreed == 1) + free(pRes); + *pbMustBeFreed = 0; + return "**OUT OF MEMORY**"; + } + pSrc = pRes; + *pDst++ = '"'; /* starting quote */ + while(*pSrc) { + if(*pSrc == '"') + *pDst++ = '"'; /* need to add double double quote (see RFC4180) */ + *pDst++ = *pSrc++; + } + *pDst++ = '"'; /* ending quote */ + *pDst = '\0'; + if(*pbMustBeFreed == 1) + free(pRes); + pRes = pBStart; + *pbMustBeFreed = 1; + } + /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ return(pRes); } diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 21d284f3..8dddc4b8 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -141,5 +141,23 @@ int OMSRgetEntry(omodStringRequest_t *pThis, int iEntry, uchar **ppTplName, int return RS_RET_OK; } + + +/* return the full set of template options that are supported by this version of + * OMSR. They are returned in an unsigned long value. The caller can mask that + * value to check on the option he is interested in. + * Note that this interface was added in 4.1.6, so a plugin must obtain a pointer + * to this interface via queryHostEtryPt(). + * rgerhards, 2009-04-03 + */ +rsRetVal +OMSRgetSupportedTplOpts(unsigned long *pOpts) +{ + DEFiRet; + assert(pOpts != NULL); + *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY; + RETiRet; +} + /* vim:set ai: */ diff --git a/runtime/objomsr.h b/runtime/objomsr.h index 2255e4f3..75ad0fb8 100644 --- a/runtime/objomsr.h +++ b/runtime/objomsr.h @@ -1,6 +1,6 @@ /* Definition of the omsr (omodStringRequest) object. * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -27,7 +27,8 @@ /* define flags for required template options */ #define OMSR_NO_RQD_TPL_OPTS 0 #define OMSR_RQD_TPL_OPT_SQL 1 -/* next option is 2, 4, 8, ... */ +#define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */ +/* next option is 4, 8, 16, ... */ struct omodStringRequest_s { /* strings requested by output module for doAction() */ int iNumEntries; /* number of array entries for data elements below */ @@ -40,6 +41,7 @@ typedef struct omodStringRequest_s omodStringRequest_t; rsRetVal OMSRdestruct(omodStringRequest_t *pThis); rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries); rsRetVal OMSRsetEntry(omodStringRequest_t *pThis, int iEntry, uchar *pTplName, int iTplOpts); +rsRetVal OMSRgetSupportedTplOpts(unsigned long *pOpts); int OMSRgetEntryCount(omodStringRequest_t *pThis); int OMSRgetEntry(omodStringRequest_t *pThis, int iEntry, uchar **ppTplName, int *piTplOpts); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ac9b726f..8e376291 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -84,6 +84,8 @@ typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; +typedef struct vmstk_s vmstk_t; +typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ /* some universal 64 bit define... */ typedef long long int64; @@ -258,9 +260,12 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ - RS_RET_ERR_CREAT_PIPE = -2114, /**< error during pipe creation */ - RS_RET_ERR_FORK = -2115, /**< error during fork() */ - RS_RET_ERR_WRITE_PIPE = -2116, /**< error writing to pipe */ + RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ + RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ + RS_RET_ERR_RLIM_NOFILE = -2116, /**< error setting max. nbr open files process limit */ + RS_RET_ERR_CREAT_PIPE = -2117, /**< error during pipe creation */ + RS_RET_ERR_FORK = -2118, /**< error during fork() */ + RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ @@ -344,6 +349,11 @@ typedef enum rsObjectID rsObjID; # define __attribute__(x) /*NOTHING*/ #endif +#ifndef O_CLOEXEC +/* of course, this limits the functionality... */ +# define O_CLOEXEC 0 +#endif + /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index a5dc625a..07256fab 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -703,17 +703,26 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) * never is a \0 *inside* a property string. * Note that the function returns -1 if regexp functionality is not available. * rgerhards: 2009-03-04: ERE support added, via parameter iType: 0 - BRE, 1 - ERE + * Arnaud Cornet/rgerhards: 2009-04-02: performance improvement by caching compiled regex + * If a caller does not need the cached version, it must still provide memory for it + * and must call rsCStrRegexDestruct() afterwards. */ -rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *rc) { - regex_t preq; + regex_t **cache = (regex_t**) rc; int ret; DEFiRet; + assert(pCS1 != NULL); + assert(psz != NULL); + assert(cache != NULL); + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); - ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); - regexp.regfree(&preq); + if (*cache == NULL) { + *cache = calloc(sizeof(regex_t), 1); + regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); + } + ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); if(ret != 0) ABORT_FINALIZE(RS_RET_NOT_FOUND); } else { @@ -725,6 +734,25 @@ finalize_it: } +/* free a cached compiled regex + * Caller must provide a pointer to a buffer that was created by + * rsCStrSzStrMatchRegexCache() + */ +void rsCStrRegexDestruct(void *rc) +{ + regex_t **cache = rc; + + assert(cache != NULL); + assert(*cache != NULL); + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + regexp.regfree(*cache); + free(*cache); + *cache = NULL; + } +} + + /* compare a rsCStr object with a classical sz string. This function * is almost identical to rsCStrZsStrCmp(), but it also takes an offset * to the CStr object from where the comparison is to start. @@ -822,7 +850,7 @@ rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber) /* TODO: octal? hex? */ n = 0; while(i < pStr->iStrLen && isdigit(pStr->pBuf[i])) { - n = n * 10 + pStr->pBuf[i] * 10; + n = n * 10 + pStr->pBuf[i] - '0'; ++i; } diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index f3e08439..684133bb 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -136,7 +136,8 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz); int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); -rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *cache); +void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); diff --git a/runtime/var.c b/runtime/var.c index 559bc56c..ef7cc8e6 100644 --- a/runtime/var.c +++ b/runtime/var.c @@ -366,7 +366,7 @@ ConvForOperation(var_t *pThis, var_t *pOther) case VARTYPE_NUMBER: /* check if we can convert pThis to a number, if so use number format. */ iRet = ConvToNumber(pThis); - if(iRet != RS_RET_NOT_A_NUMBER) { + if(iRet == RS_RET_NOT_A_NUMBER) { CHKiRet(ConvToString(pOther)); } else { FINALIZE; /* OK or error */ @@ -384,7 +384,7 @@ ConvForOperation(var_t *pThis, var_t *pOther) break; case VARTYPE_STR: iRet = ConvToNumber(pOther); - if(iRet != RS_RET_NOT_A_NUMBER) { + if(iRet == RS_RET_NOT_A_NUMBER) { CHKiRet(ConvToString(pThis)); } else { FINALIZE; /* OK or error */ diff --git a/runtime/vm.c b/runtime/vm.c index a25476c2..41d3e483 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -26,6 +26,7 @@ #include "config.h" #include <stdlib.h> #include <assert.h> +#include <ctype.h> #include "rsyslog.h" #include "obj.h" @@ -39,6 +40,142 @@ DEFobjCurrIf(vmstk) DEFobjCurrIf(var) DEFobjCurrIf(sysvar) +/* ------------------------------ function registry code and structures ------------------------------ */ + +/* we maintain a registry of known functions */ +/* currently, this is a singly-linked list, this shall become a binary + * tree when we add the real call interface. So far, entries are added + * at the root, only. + */ +typedef struct s_rsf_entry { + cstr_t *pName; /* function name */ + prsf_t rsf; /* pointer to function code */ + struct s_rsf_entry *pNext; /* Pointer to next element or NULL */ +} rsf_entry_t; +rsf_entry_t *funcRegRoot = NULL; + + +/* add a function to the function registry. + * The handed-over cstr_t* object must no longer be used by the caller. + * A duplicate function name is an error. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsfrAddFunction(uchar *szName, prsf_t rsf) +{ + rsf_entry_t *pEntry; + size_t lenName; + DEFiRet; + + assert(szName != NULL); + assert(rsf != NULL); + + /* first check if we have a duplicate name, with the current approach this means + * we need to go through the whole list. + */ + lenName = strlen((char*)szName); + for(pEntry = funcRegRoot ; pEntry != NULL ; pEntry = pEntry->pNext) + if(!rsCStrSzStrCmp(pEntry->pName, szName, lenName)) + ABORT_FINALIZE(RS_RET_DUP_FUNC_NAME); + + /* unique name, so add to head of list */ + CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); + CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + pEntry->rsf = rsf; + pEntry->pNext = funcRegRoot; + funcRegRoot = pEntry; + +finalize_it: + RETiRet; +} + + +/* find a function inside the function registry + * The caller provides a cstr_t with the function name and receives + * a function pointer back. If no function is found, an RS_RET_UNKNW_FUNC + * error is returned. So if the function returns with RS_RET_OK, the caller + * can savely assume the function pointer is valid. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunction(cstr_t *pcsName, prsf_t *prsf) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(prsf != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(!rsCStrCStrCmp(pEntry->pName, pcsName)) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *prsf = pFound->rsf; + +finalize_it: + RETiRet; +} + + +/* find the name of a RainerScript function whom's function pointer + * is known. This function returns the cstr_t object, which MUST NOT + * be modified by the caller. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunctionName(prsf_t rsf, cstr_t **ppcsName) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(rsf != NULL); + assert(ppcsName != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(pEntry->rsf == rsf) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *ppcsName = pFound->pName; + +finalize_it: + RETiRet; +} + + +/* free the whole function registry + */ +static void +rsfrRemoveAll(void) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pEntryDel; + + BEGINfunc + pEntry = funcRegRoot; + while(pEntry != NULL) { + pEntryDel = pEntry; + pEntry = pEntry->pNext; + rsCStrDestruct(&pEntryDel->pName); + free(pEntryDel); + } + funcRegRoot = NULL; + ENDfunc +} + + +/* ------------------------------ end function registry code and structures ------------------------------ */ + /* ------------------------------ instruction set implementation ------------------------------ * * The following functions implement the VM's instruction set. @@ -330,7 +467,6 @@ CODESTARTop(PUSHSYSVAR) finalize_it: ENDop(PUSHSYSVAR) - /* The function call operation is only very roughly implemented. While the plumbing * to reach this instruction is fine, the instruction itself currently supports only * functions with a single argument AND with a name that we know. @@ -340,20 +476,9 @@ ENDop(PUSHSYSVAR) */ BEGINop(FUNC_CALL) /* remember to set the instruction also in the ENDop macro! */ var_t *numOperands; - var_t *operand1; - int iStrlen; CODESTARTop(FUNC_CALL) vmstk.PopNumber(pThis->pStk, &numOperands); - if(numOperands->val.num != 1) - ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); - vmstk.PopString(pThis->pStk, &operand1); /* guess there's just one ;) */ - if(!rsCStrSzStrCmp(pOp->operand.pVar->val.pStr, (uchar*) "strlen", 6)) { /* only one supported so far ;) */ -RUNLOG_VAR("%s", rsCStrGetSzStr(operand1->val.pStr)); - iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); -RUNLOG_VAR("%d", iStrlen); - } else - ABORT_FINALIZE(RS_RET_INVLD_FUNC); - PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME + CHKiRet((*pOp->operand.rsf)(pThis->pStk, numOperands->val.num)); var.Destruct(&numOperands); /* no longer needed */ finalize_it: ENDop(FUNC_CALL) @@ -362,6 +487,89 @@ ENDop(FUNC_CALL) /* ------------------------------ end instruction set implementation ------------------------------ */ +/* ------------------------------ begin built-in function implementation ------------------------------ */ +/* note: this shall probably be moved to a separate module, but for the time being we do it directly + * in here. This is on our way to get from a dirty to a clean solution via baby steps that are + * a bit less dirty each time... + * + * The advantage of doing it here is that we do not yet need to think about how to handle the + * exit case, where we must not unload function modules which functions are still referenced. + * + * CALLING INTERFACE: + * The function must pop its parameters off the stack and pop its result onto + * the stack when it is finished. The number of parameters the function was + * called with is provided to it. If the argument count is less then what the function + * expected, it may handle the situation with defaults (or return an error). If the + * argument count is greater than expected, returnung an error is highly + * recommended (use RS_RET_INVLD_NBR_ARGUMENTS for these cases). + * + * All function names are prefixed with "rsf_" (RainerScript Function) to have + * a separate "name space". + * + * rgerhards, 2009-04-06 + */ + + +/* The strlen function, also probably a prototype of how all functions should be + * implemented. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_strlen(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton (trivial case here...) */ + vmstk.PopString(pStk, &operand1); + iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); + + /* Store result and cleanup */ + var.SetNumber(operand1, iStrlen); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + +/* The "tolower" function, which converts its sole argument to lower case. + * Quite honestly, currently this is primarily a test driver for me... + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_tolower(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + uchar *pSrc; + cstr_t *pcstr; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton */ + CHKiRet(rsCStrConstruct(&pcstr)); + vmstk.PopString(pStk, &operand1); + pSrc = rsCStrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); + while(iStrlen--) { + CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + } + + /* Store result and cleanup */ + CHKiRet(rsCStrFinish(pcstr)); + var.SetString(operand1, pcstr); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(vm) /* be sure to specify the object type also in END macro! */ @@ -531,10 +739,23 @@ CODESTARTobjQueryInterface(vm) pIf->PopBoolFromStack = PopBoolFromStack; pIf->PopVarFromStack = PopVarFromStack; pIf->SetMsg = SetMsg; + pIf->FindRSFunction = findRSFunction; + pIf->FindRSFunctionName = findRSFunctionName; finalize_it: ENDobjQueryInterface(vm) +/* Exit the vm class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(vm, OBJ_IS_CORE_MODULE) /* class, version */ + rsfrRemoveAll(); + objRelease(sysvar, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vmstk, CORE_COMPONENT); +ENDObjClassExit(vm) + + /* Initialize the vm class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-02-19 @@ -548,6 +769,11 @@ BEGINObjClassInit(vm, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, vmDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmConstructFinalize); + + /* register built-in functions // TODO: move to its own module */ + CHKiRet(rsfrAddFunction((uchar*)"strlen", rsf_strlen)); + CHKiRet(rsfrAddFunction((uchar*)"tolower", rsf_tolower)); + ENDObjClassInit(vm) /* vi:set ai: diff --git a/runtime/vm.h b/runtime/vm.h index d2458220..cb3c69d0 100644 --- a/runtime/vm.h +++ b/runtime/vm.h @@ -55,8 +55,11 @@ BEGINinterface(vm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*PopBoolFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*PopVarFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*SetMsg)(vm_t *pThis, msg_t *pMsg); /* there are a few cases where we need this... */ + /* v2 (4.1.7) */ + rsRetVal (*FindRSFunction)(cstr_t *pcsName, prsf_t *prsf); /* 2009-06-04 */ + rsRetVal (*FindRSFunctionName)(prsf_t rsf, cstr_t **ppcsName); /* 2009-06-04 */ ENDinterface(vm) -#define vmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmop.c b/runtime/vmop.c index a343481e..3e001d27 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -32,10 +32,12 @@ #include "rsyslog.h" #include "obj.h" #include "vmop.h" +#include "vm.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(var) +DEFobjCurrIf(vm) /* forward definitions */ @@ -61,8 +63,10 @@ rsRetVal vmopConstructFinalize(vmop_t __attribute__((unused)) *pThis) /* destructor for the vmop object */ BEGINobjDestruct(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(vmop) - if(pThis->operand.pVar != NULL) - var.Destruct(&pThis->operand.pVar); + if(pThis->opcode != opcode_FUNC_CALL) { + if(pThis->operand.pVar != NULL) + var.Destruct(&pThis->operand.pVar); + } ENDobjDestruct(vmop) @@ -72,13 +76,19 @@ BEGINobjDebugPrint(vmop) /* be sure to specify the object type also in END and C cstr_t *pStrVar; CODESTARTobjDebugPrint(vmop) vmopOpcode2Str(pThis, &pOpcodeName); - CHKiRet(rsCStrConstruct(&pStrVar)); - CHKiRet(rsCStrFinish(&pStrVar)); - if(pThis->operand.pVar != NULL) { - CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pStrVar)); + assert(pStrVar != NULL); + } else { + CHKiRet(rsCStrConstruct(&pStrVar)); + if(pThis->operand.pVar != NULL) { + CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + } } + CHKiRet(rsCStrFinish(&pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); - rsCStrDestruct(&pStrVar); + if(pThis->opcode != opcode_FUNC_CALL) + rsCStrDestruct(&pStrVar); finalize_it: ENDobjDebugPrint(vmop) @@ -98,6 +108,7 @@ static rsRetVal Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) { uchar *pOpcodeName; + cstr_t *pcsFuncName; uchar szBuf[2048]; size_t lenBuf; DEFiRet; @@ -107,8 +118,13 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) vmopOpcode2Str(pThis, &pOpcodeName); lenBuf = snprintf((char*) szBuf, sizeof(szBuf), "%s\t", pOpcodeName); CHKiRet(rsCStrAppendStrWithLen(pstrPrg, szBuf, lenBuf)); - if(pThis->operand.pVar != NULL) - CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pcsFuncName)); + CHKiRet(rsCStrAppendCStr(pstrPrg, pcsFuncName)); + } else { + if(pThis->operand.pVar != NULL) + CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + } CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); finalize_it: @@ -116,6 +132,23 @@ finalize_it: } +/* set function + * rgerhards, 2009-04-06 + */ +static rsRetVal +vmopSetFunc(vmop_t *pThis, cstr_t *pcsFuncName) +{ + prsf_t rsf; /* pointer to function */ + DEFiRet; + ISOBJ_TYPE_assert(pThis, vmop); + CHKiRet(vm.FindRSFunction(pcsFuncName, &rsf)); /* check if function exists and obtain pointer to it */ + assert(rsf != NULL); /* just double-check, would be very hard to find! */ + pThis->operand.rsf = rsf; +finalize_it: + RETiRet; +} + + /* set operand (variant case) * rgerhards, 2008-02-20 */ @@ -248,6 +281,7 @@ CODESTARTobjQueryInterface(vmop) pIf->ConstructFinalize = vmopConstructFinalize; pIf->Destruct = vmopDestruct; pIf->DebugPrint = vmopDebugPrint; + pIf->SetFunc = vmopSetFunc; pIf->SetOpcode = vmopSetOpcode; pIf->SetVar = vmopSetVar; pIf->Opcode2Str = vmopOpcode2Str; @@ -263,6 +297,7 @@ ENDobjQueryInterface(vmop) BEGINObjClassInit(vmop, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); OBJSetMethodHandler(objMethod_DEBUGPRINT, vmopDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmopConstructFinalize); diff --git a/runtime/vmop.h b/runtime/vmop.h index 938b08fd..67048c26 100644 --- a/runtime/vmop.h +++ b/runtime/vmop.h @@ -26,6 +26,7 @@ #define INCLUDED_VMOP_H #include "ctok_token.h" +#include "vmstk.h" #include "stringbuf.h" /* machine instructions types */ @@ -96,7 +97,8 @@ typedef struct vmop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ opcode_t opcode; union { - var_t *pVar; /* for function call, this is the name (string) of function to be called */ + var_t *pVar; + prsf_t rsf; /* pointer to function for "call" instruction */ } operand; struct vmop_s *pNext; /* next operation or NULL, if end of program (logically this belongs to vmprg) */ } vmop_t; @@ -112,8 +114,13 @@ BEGINinterface(vmop) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetVar)(vmop_t *pThis, var_t *pVar); rsRetVal (*Opcode2Str)(vmop_t *pThis, uchar **ppName); rsRetVal (*Obj2Str)(vmop_t *pThis, cstr_t *pstr); + /* v2 */ + rsRetVal (*SetFunc)(vmop_t *pThis, cstr_t *pcsFuncName); ENDinterface(vmop) -#define vmopCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmopCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* interface changes, v1 -> v2 + * added SetFuct after existing function pointers -- rgerhards, 2009-04-06 + */ /* the remaining prototypes */ PROTOTYPEObj(vmop); diff --git a/runtime/vmprg.c b/runtime/vmprg.c index 75915025..07757b98 100644 --- a/runtime/vmprg.c +++ b/runtime/vmprg.c @@ -155,7 +155,6 @@ vmprgAddVarOperation(vmprg_t *pThis, opcode_t opcode, var_t *pVar) /* construct and fill vmop */ CHKiRet(vmop.Construct(&pOp)); CHKiRet(vmop.ConstructFinalize(pOp)); - CHKiRet(vmop.ConstructFinalize(pOp)); CHKiRet(vmop.SetOpcode(pOp, opcode)); if(pVar != NULL) CHKiRet(vmop.SetVar(pOp, pVar)); @@ -168,6 +167,32 @@ finalize_it: } +/* this is another shortcut for high-level callers. It is similar to vmprgAddVarOperation + * but adds a call operation. Among others, this include a check if the function + * is known. + */ +static rsRetVal +vmprgAddCallOperation(vmprg_t *pThis, cstr_t *pcsName) +{ + DEFiRet; + vmop_t *pOp; + + ISOBJ_TYPE_assert(pThis, vmprg); + + /* construct and fill vmop */ + CHKiRet(vmop.Construct(&pOp)); + CHKiRet(vmop.ConstructFinalize(pOp)); + CHKiRet(vmop.SetFunc(pOp, pcsName)); + CHKiRet(vmop.SetOpcode(pOp, opcode_FUNC_CALL)); + + /* and add it to the program */ + CHKiRet(vmprgAddOperation(pThis, pOp)); + +finalize_it: + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-02-21 */ @@ -189,6 +214,7 @@ CODESTARTobjQueryInterface(vmprg) pIf->Obj2Str = Obj2Str; pIf->AddOperation = vmprgAddOperation; pIf->AddVarOperation = vmprgAddVarOperation; + pIf->AddCallOperation = vmprgAddCallOperation; finalize_it: ENDobjQueryInterface(vmprg) diff --git a/runtime/vmprg.h b/runtime/vmprg.h index c1042f7d..66f03913 100644 --- a/runtime/vmprg.h +++ b/runtime/vmprg.h @@ -57,8 +57,10 @@ BEGINinterface(vmprg) /* name must also be changed in ENDinterface macro! */ rsRetVal (*AddOperation)(vmprg_t *pThis, vmop_t *pOp); rsRetVal (*AddVarOperation)(vmprg_t *pThis, opcode_t opcode, var_t *pVar); rsRetVal (*Obj2Str)(vmprg_t *pThis, cstr_t *pstr); + /* v2 (4.1.7) */ + rsRetVal (*AddCallOperation)(vmprg_t *pThis, cstr_t *pVar); /* added 2009-04-06 */ ENDinterface(vmprg) -#define vmprgCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmprgCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmstk.h b/runtime/vmstk.h index 2d45ee4d..06657cf4 100644 --- a/runtime/vmstk.h +++ b/runtime/vmstk.h @@ -27,11 +27,11 @@ #define VMSTK_SIZE 256 /* the vmstk object */ -typedef struct vmstk_s { +struct vmstk_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ var_t *vStk[VMSTK_SIZE];/* the actual stack */ int iStkPtr; /* stack pointer, points to next free location, grows from 0 --> topend */ -} vmstk_t; +}; /* interfaces */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 9f54a9ab..04eb974f 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -206,6 +206,14 @@ wtpProcessThrdChanges(wtp_t *pThis) FINALIZE; } + /* Note: there is a left-over potential race condition below: + * pThis->bThrdStateChanged may be re-set by another thread while + * we work on it and thus the loop may terminate too early. However, + * there are no really bad effects from that so I perfer - for this + * version - to live with the problem as is. Not a good idea to + * introduce that large change into the stable branch without very + * good reason. -- rgerhards, 2009-04-02 + */ do { /* reset the change marker */ pThis->bThrdStateChanged = 0; @@ -735,6 +735,20 @@ SetDrvrPermPeers(tcpsrv_t *pThis, permittedPeers_t *pPermPeers) * -------------------------------------------------------------------------- */ +/* set max number of sessions + * this must be called before ConstructFinalize, or it will have no effect! + * rgerhards, 2009-04-09 + */ +static rsRetVal +SetSessMax(tcpsrv_t *pThis, int iMax) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->iSessMax = iMax; + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-02-29 */ @@ -762,6 +776,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetUsrP = SetUsrP; pIf->SetInputName = SetInputName; pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; + pIf->SetSessMax = SetSessMax; pIf->SetDrvrMode = SetDrvrMode; pIf->SetDrvrAuthMode = SetDrvrAuthMode; pIf->SetDrvrPermPeers = SetDrvrPermPeers; @@ -92,11 +92,14 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetCBOnSessAccept)(tcpsrv_t*, rsRetVal (*) (tcpsrv_t*, tcps_sess_t*)); rsRetVal (*SetCBOnSessDestruct)(tcpsrv_t*, rsRetVal (*) (void*)); rsRetVal (*SetCBOnSessConstructFinalize)(tcpsrv_t*, rsRetVal (*) (void*)); + /* added v5 */ + rsRetVal (*SetSessMax)(tcpsrv_t *pThis, int iMaxSess); /* 2009-04-09 */ ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 + * change for v5 and up: see above */ @@ -47,6 +47,8 @@ static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ + + /* This functions converts a template into a string. It should * actually be in template.c, but this requires larger re-structuring * of the code (because all the property-access functions are static @@ -140,6 +142,60 @@ finalize_it: RETiRet; } + +/* This functions converts a template into an array of strings. + * For further general details, see the very similar funtion + * tpltoString(). + * Instead of a string, an array of string pointers is returned by + * thus function. The caller is repsonsible for destroying that array as + * well as all of its elements. The array is of fixed size. It's end + * is indicated by a NULL pointer. + * rgerhards, 2009-04-03 + */ +rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) +{ + DEFiRet; + struct templateEntry *pTpe; + uchar **pArr; + int iArr; + unsigned short bMustBeFreed; + uchar *pVal; + + assert(pTpl != NULL); + assert(pMsg != NULL); + assert(ppArr != NULL); + + /* loop through the template. We obtain one value, create a + * private copy (if necessary), add it to the string array + * and then on to the next until we have processed everything. + */ + + CHKmalloc(pArr = calloc(pTpl->tpenElements + 1, sizeof(uchar*))); + iArr = 0; + + pTpe = pTpl->pEntryRoot; + while(pTpe != NULL) { + if(pTpe->eEntryType == CONSTANT) { + CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); + } else if(pTpe->eEntryType == FIELD) { + pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); + if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ + pArr[iArr] = pVal; /* ... so we can use it! */ + } else { + CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pVal)); + } + } + iArr++; + pTpe = pTpe->pNext; + } + +finalize_it: + *ppArr = (iRet == RS_RET_OK) ? pArr : NULL; + + RETiRet; +} + + /* Helper to doSQLEscape. This is called if doSQLEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can @@ -460,6 +516,8 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) pTpe->data.field.options.bSecPathDrop = 1; } else if(!strcmp((char*)Buf, "secpath-replace")) { pTpe->data.field.options.bSecPathReplace = 1; + } else if(!strcmp((char*)Buf, "csv")) { + pTpe->data.field.options.bCSV = 1; } else { dbgprintf("Invalid field option '%s' specified - ignored.\n", Buf); } @@ -1105,6 +1163,9 @@ void tplPrintList(void) if(pTpe->data.field.options.bSPIffNo1stSP) { dbgprintf("[SP iff no first SP] "); } + if(pTpe->data.field.options.bCSV) { + dbgprintf("[format as CSV (RFC4180)]"); + } if(pTpe->data.field.options.bDropLastLF) { dbgprintf("[drop last LF in msg] "); } @@ -94,9 +94,10 @@ struct templateEntry { unsigned bSpaceCC: 1; /* change control characters to spaceescape? */ unsigned bEscapeCC: 1; /* escape control characters? */ unsigned bDropLastLF: 1; /* drop last LF char in msg (PIX!) */ - unsigned bSecPathDrop: 1; /* drop slashes, replace dots, empty string */ - unsigned bSecPathReplace: 1; /* replace slashes, replace dots, empty string */ - unsigned bSPIffNo1stSP: 1; /* replace slashes, replace dots, empty string */ + unsigned bSecPathDrop: 1; /* drop slashes, replace dots, empty string */ + unsigned bSecPathReplace: 1; /* replace slashes, replace dots, empty string */ + unsigned bSPIffNo1stSP: 1; /* replace slashes, replace dots, empty string */ + unsigned bCSV: 1; /* format field in CSV (RFC 4180) format */ } options; /* options as bit fields */ } field; } data; @@ -125,6 +126,7 @@ void tplLastStaticInit(struct template *tpl); * BEFORE msg.h, even if your code file does not actually need it. * rgerhards, 2007-08-06 */ +rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz); rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); diff --git a/tests/3.rstest b/tests/3.rstest index 93cb941a..e75d9754 100644 --- a/tests/3.rstest +++ b/tests/3.rstest @@ -7,10 +7,10 @@ out: 00000000: push_msgvar msg[cstr] 00000001: push_const abc[cstr] 00000002: push_const 1[nbr] -00000003: func_call strlen[cstr] +00000003: func_call strlen 00000004: strconcat 00000005: push_const 1[nbr] -00000006: func_call strlen[cstr] +00000006: func_call strlen 00000007: push_const 20[nbr] 00000008: push_const 30[nbr] 00000009: add diff --git a/tests/Makefile.am b/tests/Makefile.am index 09d1a0b6..87dca985 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ -TESTRUNS = rt_init rscript parsertest -check_PROGRAMS = $(TESTRUNS) ourtail -TESTS = $(TESTRUNS) cfg.sh +TESTRUNS = rt_init rscript +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -17,19 +17,29 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ - testruns/parser.conf \ - testruns/1.parse1 \ - testruns/rfc3164.parse1 \ - testruns/rfc5424-1.parse1 \ - testruns/rfc5424-2.parse1 \ - testruns/rfc5424-3.parse1 \ - testruns/rfc5424-4.parse1 \ + testsuites/parse1.conf \ + testsuites/1.parse1 \ + testsuites/rfc3164.parse1 \ + testsuites/rfc5424-1.parse1 \ + testsuites/rfc5424-2.parse1 \ + testsuites/rfc5424-3.parse1 \ + testsuites/rfc5424-4.parse1 \ + testsuites/omod-if-array.conf \ + testsuites/1.omod-if-array \ + parsertest.sh \ + manytcp.sh \ + testsuites/manytcp.conf \ + omod-if-array.sh \ cfg.sh ourtail_SOURCES = ourtail.c +chkseq_SOURCES = chkseq.c -parsertest_SOURCES = parsertest.c getline.c -parsertest_LDADD = $(SOL_LIBS) +tcpflood_SOURCES = tcpflood.c +tcpflood_LDADD = $(SOL_LIBS) + +nettester_SOURCES = nettester.c getline.c +nettester_LDADD = $(SOL_LIBS) rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/chkseq.c b/tests/chkseq.c new file mode 100644 index 00000000..3203c250 --- /dev/null +++ b/tests/chkseq.c @@ -0,0 +1,76 @@ +/* Checks if a file consists of line of strictly monotonically + * increasing numbers. An expected start and end number may + * be set. + * + * Params + * argv[1] file to check + * argv[2] start number + * argv[3] end number + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char *argv[]) +{ + FILE *fp; + int val; + int i; + int ret = 0; + int start, end; + + if(argc != 4) { + printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file start end\n"); + exit(1); + } + + start = atoi(argv[2]); + end = atoi(argv[3]); + + if(start > end) { + printf("start must be less than or equal end!\n"); + exit(1); + } + + /* read file */ + fp = fopen(argv[1], "r"); + if(fp == NULL) { + perror(argv[1]); + exit(1); + } + + for(i = start ; i < end ; ++i) { + if(fscanf(fp, "%d\n", &val) != 1) { + printf("scanf error in index i=%d\n", i); + exit(1); + } + if(val != i) { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } + } + + exit(ret); +} diff --git a/tests/getline.c b/tests/getline.c index 10de2ffe..617d1b0e 100644 --- a/tests/getline.c +++ b/tests/getline.c @@ -23,7 +23,8 @@ */ #include "config.h" #include <stdio.h> -#include <malloc.h> +#include <sys/types.h> +#include <stdlib.h> /* we emulate getline (the dirty way) if we do not have it * We do not try very hard, as this is just a test driver. diff --git a/tests/manytcp.sh b/tests/manytcp.sh new file mode 100755 index 00000000..d9b2e9a0 --- /dev/null +++ b/tests/manytcp.sh @@ -0,0 +1,21 @@ +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# the config file specifies exactly 1100 connections +./tcpflood 127.0.0.1 13514 1000 40000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 5 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 39999 +if [ "$?" -ne "0" ]; then + rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log diff --git a/tests/parsertest.c b/tests/nettester.c index 6c2221e8..37183ac9 100644 --- a/tests/parsertest.c +++ b/tests/nettester.c @@ -1,11 +1,14 @@ -/* Runs a test suite on the rsyslog parser (and later potentially +/* Runs a test suite on the rsyslog (and later potentially * other things). * - * Please note that this - * program works together with the config file AND easily extensible - * test case files (*.parse1) to run a number of checks. All test - * cases are executed, even if there is a failure early in the - * process. When finished, the numberof failed tests will be given. + * The name of the test suite must be given as argv[1]. In this config, + * rsyslogd is loaded with config ./testsuites/<name>.conf and then + * test cases ./testsuites/ *.<name> are executed on it. This test driver is + * suitable for testing cases where a message sent (via UDP) results in + * exactly one response. It can not be used in cases where no response + * is expected (that would result in a hang of the test driver). + * Note: each test suite can contain many tests, but they all need to work + * with the same rsyslog configuration. * * Part of the testbench for rsyslog. * @@ -40,9 +43,18 @@ #include <unistd.h> #include <string.h> #include <glob.h> +#include <signal.h> #include <netinet/in.h> #define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static enum { inputUDP, inputTCP } inputMode; /* input for which tests are to be run */ +static pid_t rsyslogdPid = 0; /* pid of rsyslog instance being tested */ +static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ +static char *testSuite; /* name of current test suite */ void readLine(int fd, char *ln) @@ -58,6 +70,59 @@ void readLine(int fd, char *ln) } +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + /* send a message via UDP * returns 0 if ok, something else otherwise. */ @@ -90,22 +155,23 @@ udpSend(char *buf, int lenBuf) return 0; } + /* open pipe to test candidate - so far, this is * always rsyslogd and with a fixed config. Later, we may * change this. Returns 0 if ok, something else otherwise. * rgerhards, 2009-03-31 */ -int openPipe(pid_t *pid, int *pfd) +int openPipe(char *configFile, pid_t *pid, int *pfd) { int pipefd[2]; pid_t cpid; char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime//.libs", NULL }; + "-M../runtime/.libs:../.libs", NULL }; char confFile[1024]; char *newenviron[] = { NULL }; - sprintf(confFile, "-f%s/testruns/parser.conf", getenv("srcdir")); + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); newargv[1] = confFile; if (pipe(pipefd) == -1) { @@ -166,9 +232,14 @@ processTestFile(int fd, char *pszFileName) testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send */ - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); + /* now we have the test data to send (we could use function pointers here...) */ + if(inputMode == inputUDP) { + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + } else { + if(tcpSend(testdata, strlen(testdata)) != 0) + return(2); + } /* next line is expected output * we do not care about EOF here, this will lead to a failure and thus @@ -242,28 +313,82 @@ doTests(int fd, char *files) return(iFailed); } +/* cleanup */ +void doAtExit(void) +{ + int status; + + if(rsyslogdPid != 0) { + kill(rsyslogdPid, SIGTERM); + waitpid(rsyslogdPid, &status, 0); /* wait until instance terminates */ + } -/* */ + unlink(NETTEST_INPUT_CONF_FILE); +} + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ int main(int argc, char *argv[]) { int fd; - pid_t pid; int ret = 0; + FILE *fp; char buf[4096]; char testcases[4096]; - printf("running rsyslog parser tests ($srcdir=%s)\n", getenv("srcdir")); + if(argc != 3) { + printf("Invalid call of nettester\n"); + printf("Usage: nettester testsuite-name input\n"); + printf(" input = udp|tcp\n"); + exit(1); + } + + atexit(doAtExit); + + testSuite = argv[1]; + + if(!strcmp(argv[2], "udp")) + inputMode = inputUDP; + else if(!strcmp(argv[2], "tcp")) + inputMode = inputTCP; + else { + printf("error: unsupported input mode '%s'\n", argv[2]); + exit(1); + } + + if((srcdir = getenv("srcdir")) == NULL) + srcdir = "."; + + printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s)\n", + srcdir, testSuite, argv[2]); + + /* create input config file */ + if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { + perror(NETTEST_INPUT_CONF_FILE); + printf("error opening input configuration file\n"); + exit(1); + } + if(inputMode == inputUDP) { + fputs("$ModLoad ../plugins/imudp/.libs/imudp\n", fp); + fputs("$UDPServerRun 12514\n", fp); + } else { + fputs("$ModLoad ../plugins/imtcp/.libs/imtcp\n", fp); + fputs("$InputTCPServerRun 13514\n", fp); + } + fclose(fp); - openPipe(&pid, &fd); + /* start to be tested rsyslogd */ + openPipe(argv[1], &rsyslogdPid, &fd); readLine(fd, buf); /* generate filename */ - sprintf(testcases, "%s/testruns/*.parse1", getenv("srcdir")); + sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); if(doTests(fd, testcases) != 0) ret = 1; - /* cleanup */ - kill(pid, SIGTERM); - printf("End of parser tests.\n"); + printf("End of nettester run (%d).\n", ret); exit(ret); } diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh new file mode 100755 index 00000000..fd845b4d --- /dev/null +++ b/tests/omod-if-array.sh @@ -0,0 +1,12 @@ +echo test omod-if-array via udp +./nettester omod-if-array udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test omod-if-array via tcp +./nettester omod-if-array tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi + diff --git a/tests/parsertest.sh b/tests/parsertest.sh new file mode 100755 index 00000000..a6b7d45c --- /dev/null +++ b/tests/parsertest.sh @@ -0,0 +1,11 @@ +echo test parsertest via udp +./nettester parse1 udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test parsertest via tcp +./nettester parse1 tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/tcpflood.c b/tests/tcpflood.c new file mode 100644 index 00000000..254e9fd6 --- /dev/null +++ b/tests/tcpflood.c @@ -0,0 +1,282 @@ +/* Opens a large number of tcp connections and sends + * messages over them. This is used for stress-testing. + * + * Params + * argv[1] target address + * argv[2] target port + * argv[3] number of connections + * argv[4] number of messages to send (connection is random) + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <arpa/inet.h> +#include <assert.h> +#include <unistd.h> +#include <string.h> +#include <netinet/in.h> + +#define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static char *targetIP; +static int targetPort; +static int numMsgsToSend; /* number of messages to send */ +static int numConnections; /* number of connections to create */ +static int *sockArray; /* array of sockets to use */ + + +/* open a single tcp connection + */ +int openConn(int *fd) +{ + int sock; + struct sockaddr_in addr; + + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(targetPort); + if(inet_aton(targetIP, &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + perror("connect()"); + fprintf(stderr, "connect() failed\n"); + return(1); + } + + *fd = sock; + return 0; +} + + +/* open all requested tcp connections + * this includes allocating the connection array + */ +int openConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " open connections", sizeof(" open connections")-1); + sockArray = calloc(numConnections, sizeof(int)); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + printf("\r%5.5d", i); + //lenMsg = sprintf(msgBuf, "\r%5.5d", i); + //write(1, msgBuf, lenMsg); + } + if(openConn(&(sockArray[i])) != 0) { + printf("error in trying to open connection i=%d\n", i); + return 1; + } + } + lenMsg = sprintf(msgBuf, "\r%5.5d open connections\n", i); + write(1, msgBuf, lenMsg); + + return 0; +} + + +/* we also close all connections because otherwise we may get very bad + * timing for the syslogd - it may not be able to process all incoming + * messages fast enough if we immediately shut down. + * TODO: it may be an interesting excercise to handle that situation + * at the syslogd level, too + * rgerhards, 2009-04-14 + */ +void closeConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " close connections", sizeof(" close connections")-1); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + lenMsg = sprintf(msgBuf, "\r%5.5d", i); + write(1, msgBuf, lenMsg); + } + close(sockArray[i]); + } + lenMsg = sprintf(msgBuf, "\r%5.5d close connections\n", i); + write(1, msgBuf, lenMsg); + +} + + +/* send messages to the tcp connections we keep open. We use + * a very basic format that helps identify the message + * (via msgnum:<number>: e.g. msgnum:00000001:). This format is suitable + * for extracton to field-based properties. + * The first numConnection messages are sent sequentially, as are the + * last. All messages in between are sent over random connections. + * Note that message numbers start at 0. + */ +int sendMessages(void) +{ + int i; + int socknum; + int lenBuf; + int lenSend; + char buf[2048]; + char msgBuf[128]; + size_t lenMsg; + + srand(time(NULL)); /* seed is good enough for our needs */ + + printf("Sending %d messages.\n", numMsgsToSend); + printf("\r%5.5d messages sent", 0); + lenMsg = sprintf(msgBuf, "\r%5.5d/%5.5d messages sent", 0, numMsgsToSend); + write(1, msgBuf, lenMsg); + for(i = 0 ; i < numMsgsToSend ; ++i) { + if(i < numConnections) + socknum = i; + else if(i >= numMsgsToSend - numConnections) + socknum = i - (numMsgsToSend - numConnections); + else + socknum = rand() % numConnections; + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + lenSend = send(sockArray[socknum], buf, lenBuf, MSG_NOSIGNAL); + if(lenSend != lenBuf) { + printf("\r%5.5d\n", i); + fflush(stdout); + perror("send test data"); + printf("send() failed at socket %d, index %d\n", socknum, i); + fflush(stderr); + return(1); + } + if(i % 100 == 0) { + printf("\r%5.5d", i); + } + } + printf("\r%5.5d messages sent\n", i); + + return 0; +} + + +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int ret = 0; + static char buf[1024]; + + setvbuf(stdout, buf, _IONBF, 48); + + if(argc != 5) { + printf("Invalid call of tcpflood\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); + exit(1); + } + + targetIP = argv[1]; + targetPort = atoi(argv[2]); + numConnections = atoi(argv[3]); + numMsgsToSend = atoi(argv[4]); + + if(openConnections() != 0) { + printf("error opening connections\n"); + exit(1); + } + + if(sendMessages() != 0) { + printf("error sending messages\n"); + exit(1); + } + + //closeConnections(); + printf("End of tcpflood Run\n"); + + exit(ret); +} diff --git a/tests/testsuites/1.omod-if-array b/tests/testsuites/1.omod-if-array new file mode 100644 index 00000000..c464b19c --- /dev/null +++ b/tests/testsuites/1.omod-if-array @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, diff --git a/tests/testruns/1.parse1 b/tests/testsuites/1.parse1 index 5ae655e6..5ae655e6 100644 --- a/tests/testruns/1.parse1 +++ b/tests/testsuites/1.parse1 diff --git a/tests/testsuites/2.parse1 b/tests/testsuites/2.parse1 new file mode 100644 index 00000000..628e06df --- /dev/null +++ b/tests/testsuites/2.parse1 @@ -0,0 +1,3 @@ +<38>Mar 27 19:06:53 source_server sshd(pam_unix)[12750]: session opened for user foo by (uid=0) +38,auth,info,Mar 27 19:06:53,source_server,sshd(pam_unix),sshd(pam_unix)[12750]:, session opened for user foo by (uid=0) +# yet another real-life sample where we had some issues with diff --git a/tests/testsuites/3.parse1 b/tests/testsuites/3.parse1 new file mode 100644 index 00000000..a6b4e884 --- /dev/null +++ b/tests/testsuites/3.parse1 @@ -0,0 +1,3 @@ +<38>Apr 6 15:07:10 lxcvs07 sshd(pam_unix)[31738]: session closed for user cvsadmin +38,auth,info,Apr 6 15:07:10,lxcvs07,sshd(pam_unix),sshd(pam_unix)[31738]:, session closed for user cvsadmin +# yet another real-life sample where we had some issues with diff --git a/tests/testsuites/date1.parse1 b/tests/testsuites/date1.parse1 new file mode 100644 index 00000000..ffc7c373 --- /dev/null +++ b/tests/testsuites/date1.parse1 @@ -0,0 +1,3 @@ +<38> Mar 7 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# one space in front of the date diff --git a/tests/testsuites/date2.parse1 b/tests/testsuites/date2.parse1 new file mode 100644 index 00000000..8d587d9d --- /dev/null +++ b/tests/testsuites/date2.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# only one space between "Mar" and "7" diff --git a/tests/testsuites/date3.parse1 b/tests/testsuites/date3.parse1 new file mode 100644 index 00000000..940d261e --- /dev/null +++ b/tests/testsuites/date3.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 2008 19:06:53: example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# the year should not be there, nor the colon after the date, but we accept it... diff --git a/tests/testsuites/date4.parse1 b/tests/testsuites/date4.parse1 new file mode 100644 index 00000000..eee5fb09 --- /dev/null +++ b/tests/testsuites/date4.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 2008 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# the year should not be there, but we accept it... diff --git a/tests/testsuites/date5.parse1 b/tests/testsuites/date5.parse1 new file mode 100644 index 00000000..be32e605 --- /dev/null +++ b/tests/testsuites/date5.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 19:06:53: example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# colon after timestamp is strictly not ok, but we accept it diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf new file mode 100644 index 00000000..8175732e --- /dev/null +++ b/tests/testsuites/manytcp.conf @@ -0,0 +1,13 @@ +# Test for tcp "flood" testing +# rgerhards, 2009-04-08 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$MaxOpenFiles 2000 +$InputTCPMaxSessions 1100 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/omod-if-array.conf b/tests/testsuites/omod-if-array.conf new file mode 100644 index 00000000..d88db166 --- /dev/null +++ b/tests/testsuites/omod-if-array.conf @@ -0,0 +1,13 @@ +# Test config for array-passing output module interface +# (stanard string passing is already tested via the other test inside +# the testbench, so we do not need to focus on that) +# rgerhards, 2009-04-03 +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ActionOMStdoutArrayInterface on +$ErrorMessagesToStderr off + +# do NOT remove \n, that would hang the test driver! +$template expect,"%PRI%%timestamp%%hostname%%programname%%syslogtag%\n" +*.* :omstdout:;expect diff --git a/tests/testruns/parser.conf b/tests/testsuites/parse1.conf index 0fb7d16d..947a05a8 100644 --- a/tests/testruns/parser.conf +++ b/tests/testsuites/parse1.conf @@ -1,6 +1,5 @@ $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ErrorMessagesToStderr off diff --git a/tests/testruns/rfc3164.parse1 b/tests/testsuites/rfc3164.parse1 index e7a5fa18..e7a5fa18 100644 --- a/tests/testruns/rfc3164.parse1 +++ b/tests/testsuites/rfc3164.parse1 diff --git a/tests/testruns/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1 index 23836c9f..23836c9f 100644 --- a/tests/testruns/rfc5424-1.parse1 +++ b/tests/testsuites/rfc5424-1.parse1 diff --git a/tests/testruns/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1 index a86fbc35..a86fbc35 100644 --- a/tests/testruns/rfc5424-2.parse1 +++ b/tests/testsuites/rfc5424-2.parse1 diff --git a/tests/testruns/rfc5424-3.parse1 b/tests/testsuites/rfc5424-3.parse1 index 6ad4073d..6ad4073d 100644 --- a/tests/testruns/rfc5424-3.parse1 +++ b/tests/testsuites/rfc5424-3.parse1 diff --git a/tests/testruns/rfc5424-4.parse1 b/tests/testsuites/rfc5424-4.parse1 index ecf27e14..ecf27e14 100644 --- a/tests/testruns/rfc5424-4.parse1 +++ b/tests/testsuites/rfc5424-4.parse1 diff --git a/tools/omfile.c b/tools/omfile.c index 5141b4da..36f160d1 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -301,7 +301,7 @@ int resolveFileSizeLimit(instanceData *pData) free(pCmd); - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); actualFileSize = lseek(pData->fd, 0, SEEK_END); @@ -394,13 +394,13 @@ prepareFile(instanceData *pData, uchar *newFileName) { DEFiRet; if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); + pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); FINALIZE; /* we are done in this case */ } if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); } else { pData->fd = -1; @@ -419,7 +419,7 @@ prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); if(pData->fd != -1) { /* check and set uid/gid */ @@ -653,7 +653,7 @@ again: && e == EBADF #endif ) { - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY); + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC); if (pData->fd < 0) { iRet = RS_RET_SUSPENDED; errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); @@ -742,7 +742,7 @@ CODESTARTparseSelectorAct pData->bDynamicName = 0; pData->fCreateMode = fCreateMode; /* preserve current setting */ pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); } break; diff --git a/tools/pidfile.c b/tools/pidfile.c index 2be13da6..e7744513 100644 --- a/tools/pidfile.c +++ b/tools/pidfile.c @@ -97,7 +97,7 @@ int write_pid (char *pidfile) int fd; int pid; - if ( ((fd = open(pidfile, O_RDWR|O_CREAT, 0644)) == -1) + if ( ((fd = open(pidfile, O_RDWR|O_CREAT|O_CLOEXEC, 0644)) == -1) || ((f = fdopen(fd, "r+")) == NULL) ) { fprintf(stderr, "Can't open or create %s.\n", pidfile); return 0; diff --git a/tools/syslogd.c b/tools/syslogd.c index c72160fb..8c86c12e 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -87,6 +87,7 @@ #include <sys/ioctl.h> #include <sys/wait.h> #include <sys/file.h> +#include <sys/resource.h> #include <grp.h> #if HAVE_SYS_TIMESPEC_H @@ -429,6 +430,8 @@ selectorDestruct(void *pVal) rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); if(pThis->f_filterData.prop.pCSCompValue != NULL) rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); + if(pThis->f_filterData.prop.regex_cache != NULL) + rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); } else if(pThis->f_filter_type == FILTER_EXPR) { if(pThis->f_filterData.f_expr != NULL) expr.Destruct(&pThis->f_filterData.f_expr); @@ -549,7 +552,7 @@ void untty(void) int i; if ( !Debug ) { - i = open(_PATH_TTY, O_RDWR); + i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) (void) ioctl(i, (int) TIOCNOTTY, (char *)0); @@ -1076,12 +1079,12 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce break; case FIOP_REGEX: if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0) == RS_RET_OK) + (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) bRet = 1; break; case FIOP_EREREGEX: if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1) == RS_RET_OK) + (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) bRet = 1; break; default: @@ -1494,7 +1497,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the fields. I think this logic shall work with any type of syslog message. */ bTAGCharDetected = 0; - if(pMsg->bParseHOSTNAME) { + if(flags & PARSE_HOSTNAME) { /* TODO: quick and dirty memory allocation */ /* the memory allocated is far too much in most cases. But on the plus side, * it is quite fast... - rgerhards, 2007-09-20 @@ -2071,6 +2074,32 @@ static rsRetVal setActionResumeInterval(void __attribute__((unused)) *pVal, int } +/* set the processes max number ob files (upon configuration request) + * 2009-04-14 rgerhards + */ +static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) +{ + struct rlimit maxFiles; + char errStr[1024]; + DEFiRet; + + maxFiles.rlim_cur = iFiles; + maxFiles.rlim_max = iFiles; + + if(setrlimit(RLIMIT_NOFILE, &maxFiles) < 0) { + /* NOTE: under valgrind, we seem to be unable to extend the size! */ + rs_strerror_r(errno, errStr, sizeof(errStr)); + errmsg.LogError(0, RS_RET_ERR_RLIM_NOFILE, "could not set process file limit to %d: %s [kernel max %ld]", + iFiles, errStr, (long) maxFiles.rlim_max); + ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); + } + dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); + +finalize_it: + RETiRet; +} + + /* set the processes umask (upon configuration request) */ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { @@ -2868,6 +2897,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"modload", 0, eCmdHdlrCustomHandler, conf.doModLoad, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"includeconfig", 0, eCmdHdlrCustomHandler, conf.doIncludeLine, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"umask", 0, eCmdHdlrFileCreateMode, setUmask, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"maxopenfiles", 0, eCmdHdlrInt, setMaxFiles, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprinttemplatelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintTemplateList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintmodulelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintModuleList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintcfsyslinehandlerlist", 0, eCmdHdlrBinary, @@ -3089,6 +3119,7 @@ GlobalClassExit(void) objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ objRelease(conf, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); + vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ objRelease(vm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); @@ -3588,18 +3619,10 @@ int realMain(int argc, char **argv) fprintf(stderr, "-t option only supported in compatibility modes 0 to 2 - ignored\n"); break; case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ -{ -char buf[1024]; -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -printf("chroot to '%s'\n", arg); if(chroot(arg) != 0) { perror("chroot"); exit(1); } -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -} break; case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); diff --git a/tools/syslogd.h b/tools/syslogd.h index f1b11a91..8b9bd131 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -73,6 +73,7 @@ struct filed { FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } operation; + regex_t *regex_cache; /* cache for compiled REs, if such are used */ cstr_t *pCSCompValue; /* value to "compare" against */ char isNegated; /* actually a boolean ;) */ } prop; |