From 51e690f720b4a53a431e7a536b2fe8c25e866f7d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Mar 2009 18:54:42 +0100 Subject: fixed some portability issues first noticed on FreeBSD Also, some cosmetic improvements. --- plugins/omstdout/omstdout.c | 2 +- tests/getline.c | 3 ++- tests/nettester.c | 4 +++- tests/omod-if-array.sh | 9 ++++++++- tests/parsertest.sh | 8 +++++++- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 7c63b5c4..181895a4 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -110,7 +110,7 @@ CODESTARTdoAction if(iParam > 0) szBuf[iBuf++] = ','; /* all but first need a delimiter */ iParamVal = 0; - while(szParams[iParam][iParamVal] != '\0' && iBuf < sizeof(szBuf)) { + while(szParams[iParam][iParamVal] != '\0' && iBuf < (int) sizeof(szBuf)) { szBuf[iBuf++] = szParams[iParam][iParamVal++]; } ++iParam; diff --git a/tests/getline.c b/tests/getline.c index 10de2ffe..617d1b0e 100644 --- a/tests/getline.c +++ b/tests/getline.c @@ -23,7 +23,8 @@ */ #include "config.h" #include -#include +#include +#include /* we emulate getline (the dirty way) if we do not have it * We do not try very hard, as this is just a test driver. diff --git a/tests/nettester.c b/tests/nettester.c index 89a784f3..37183ac9 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #define EXIT_FAILURE 1 @@ -361,7 +362,8 @@ int main(int argc, char *argv[]) if((srcdir = getenv("srcdir")) == NULL) srcdir = "."; - printf("Start of nettester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s)\n", + srcdir, testSuite, argv[2]); /* create input config file */ if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index 8a8d67f3..fd845b4d 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,5 +1,12 @@ -#!/bin/bash -e echo test omod-if-array via udp ./nettester omod-if-array udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + echo test omod-if-array via tcp ./nettester omod-if-array tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi + diff --git a/tests/parsertest.sh b/tests/parsertest.sh index fabe7e8d..a6b7d45c 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,5 +1,11 @@ -#!/bin/bash -e echo test parsertest via udp ./nettester parse1 udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + echo test parsertest via tcp ./nettester parse1 tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi -- cgit From 97480eafbc67ec7e84497868a1777ce0d7881e6c Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:24 +0100 Subject: Start the output module for Oracle. Currently, resources are allocated, freed and the code compiles. No tests yet. --- plugins/omoracle/Makefile.am | 8 +++ plugins/omoracle/omoracle.c | 139 +++++++++++++++++++++++++++++++++++++++++++ plugins/omoracle/omoracle.h | 23 +++++++ 3 files changed, 170 insertions(+) create mode 100644 plugins/omoracle/Makefile.am create mode 100644 plugins/omoracle/omoracle.c create mode 100644 plugins/omoracle/omoracle.h diff --git a/plugins/omoracle/Makefile.am b/plugins/omoracle/Makefile.am new file mode 100644 index 00000000..6b75218f --- /dev/null +++ b/plugins/omoracle/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omoracle.la + +omoracle_la_SOURCES = omoracle.c omoracle.h +omoracle_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(ORACLE_CFLAGS) +omoracle_la_LDFLAGS = -module -avoid-version +omoracle_la_LIBADD = $(ORACLE_LIBS) + +EXTRA_DIST = diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c new file mode 100644 index 00000000..696cd908 --- /dev/null +++ b/plugins/omoracle/omoracle.c @@ -0,0 +1,139 @@ +/** omoracle.c + + This is an output module feeding directly to an Oracle + database. It uses Oracle Call Interface, a propietary module + provided by Oracle. + + Author: Luis Fernando Muñoz Mejías + + + This file is part of rsyslog. +*/ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" +#include "omoracle.h" + +MODULE_TYPE_OUTPUT + +/** */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +typedef struct _instanceData { + OCIEnv* environment; + OCISession* session; + OCIError* error; + OCIServer* server; + OCIStmt* statement; + OCISvcCtx* service; + OCIAuthInfo* authinfo; + OCIBind* binding; +} instanceData; + +/** Generic function for handling errors from OCI. + + It will be called only inside CHECKERR and CHECKENV macros. + + Arguments: handle The error or environment handle to check. + htype: OCI_HTYPE_* constant, usually OCI_HTYPE_ERROR or + OCI_HTYPE_ENV + status: status code to check, usually the return value of an OCI + function. + + Returns OCI_SUCCESS on success, OCI_ERROR otherwise. +*/ +static int oci_errors(void* handle, ub4 htype, sword status) +{ + sb4 errcode; + char buf[MAX_BUFSIZE]; + + switch (status) { + case OCI_SUCCESS: + return OCI_SUCCESS; + break; + case OCI_SUCCESS_WITH_INFO: + printf ("OCI SUCCESS - With info\n"); + break; + case OCI_NEED_DATA: + printf ("OCI NEEDS MORE DATA\n"); + break; + case OCI_ERROR: + printf ("OCI GENERAL ERROR\n"); + if (handle) { + OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); + printf ("Error message: %s", buf); + } else + printf ("NULL handle\n" + "Unable to extract further information"); + break; + case OCI_INVALID_HANDLE: + printf ("OCI INVALID HANDLE\n"); + break; + case OCI_STILL_EXECUTING: + printf ("Still executing...\n"); + break; + case OCI_CONTINUE: + printf ("OCI CONTINUE\n"); + break; + } + return OCI_ERROR; +} + + +/* Resource allocation */ +BEGINcreateInstance +CODESTARTcreateInstance +CHECKENV(pData->environment, + OCIEnvCreate(&(pData->environment), OCI_DEFAULT, + NULL, NULL, NULL, NULL, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->error), + OCI_HTYPE_ERROR, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->server), + OCI_HTYPE_SERVER, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->service), + OCI_HTYPE_SVCCTX, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->authinfo), + OCI_HTYPE_AUTHINFO, 0, NULL)); +finalize_it: +ENDcreateInstance + +/** Free any resources allocated by createInstance. */ +BEGINfreeInstance +CODESTARTfreeInstance + +OCIHandleFree(pData->environment, OCI_HTYPE_ENV); +OCIHandleFree(pData->error, OCI_HTYPE_ERROR); +OCIHandleFree(pData->server, OCI_HTYPE_SERVER); +OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); +OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); + +RETiRet; + +ENDfreeInstance + +/* BEGINmodInit() */ +/* CODESTARTmodInit */ +/* *ipIFVersProvided = CURR_MOD_IF_VERSION; */ +/* CODEmodInit_QueryRegCFSLineHdlr */ +/* CHKiRet(objUse(errmsg, CORE_COMPONENT)); */ +/* ENDmodInit */ diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h new file mode 100644 index 00000000..b0e70917 --- /dev/null +++ b/plugins/omoracle/omoracle.h @@ -0,0 +1,23 @@ +/** Definitions for the Oracle output module. + + This module needs OCI to be installed (on Red Hat-like systems + this is usually the oracle-instantclient-devel RPM). + + Author: Luis Fernando Muñoz Mejías +*/ +#ifndef __OMORACLEH__ +#define __OMORACLEH__ + +/** Macros to make error handling easier. */ + +/** Checks for errors on the OCI handling. */ +#define CHECKERR(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ERROR, (status))) + +/** Checks for errors when handling the environment of OCI. */ +#define CHECKENV(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ENV, (status))) + +enum { MAX_BUFSIZE = 2048 }; + +#endif -- cgit From b8f10ad00853275045180738b91e90a80c7693cc Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:25 +0100 Subject: Include omoracle in the build system. Add configure option to build the oracle support, named --enable-oracle and fix the Makefile.am accordingly. --- Makefile.am | 7 ++++++- configure.ac | 36 +++++++++++++++++++++++++++++++++++- plugins/omoracle/Makefile.am | 2 +- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 87e378ee..a5c3e267 100644 --- a/Makefile.am +++ b/Makefile.am @@ -110,15 +110,20 @@ if ENABLE_RFC3195 SUBDIRS += plugins/im3195 endif +if ENABLE_ORACLE +SUBDIRS += plugins/omoracle +endif + # tests are added as last element, because tests may need different # modules that need to be generated first SUBDIRS += tests + # make sure "make distcheck" tries to build all modules. This means that # a developer must always have an environment where every supporting library # is available. If that is not the case, the respective configure option may # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-oracle ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 0c9483ca..9be3c7ca 100644 --- a/configure.ac +++ b/configure.ac @@ -452,7 +452,39 @@ AM_CONDITIONAL(ENABLE_PGSQL, test x$enable_pgsql = xyes) AC_SUBST(PGSQL_CFLAGS) AC_SUBST(PGSQL_LIBS) - +# oracle (OCI) support +AC_ARG_ENABLE(oracle, + [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_oracle="yes" ;; + no) enable_oracle="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-oracle) ;; + esac], + [enable_oracle=no] +) +if test "x$enable_oracle" = "xyes"; then + AC_CHECK_PROG( + [HAVE_ORACLE_CONFIG], + [oracle-instantclient-config], + [yes],,, + ) + if test "x${HAVE_ORACLE_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([oracle-instantclient-config not found in PATH]) + fi + AC_CHECK_LIB( + [occi], + [OCIEnvCreate], + [ORACLE_CFLAGS="`oracle-instantclient-config --cflags`" + ORACLE_LIBS="`oracle-instantclient-config --libs`" + ], + [AC_MSG_FAILURE([Oracle (OCI) libraray is missing])], + [`oracle-instantclient-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ORACLE, test x$enable_oracle = xyes) +AC_SUBST(ORACLE_CFLAGS) +AC_SUBST(ORACLE_LIBS) + # libdbi support AC_ARG_ENABLE(libdbi, [AS_HELP_STRING([--enable-libdbi],[Enable libdbi database support @<:@default=no@:>@])], @@ -712,6 +744,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omlibdbi/Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ + plugins/omoracle/Makefile \ tests/Makefile]) AC_OUTPUT @@ -725,6 +758,7 @@ echo "Zlib compression support enabled: $enable_zlib" echo "MySql support enabled: $enable_mysql" echo "libdbi support enabled: $enable_libdbi" echo "PostgreSQL support enabled: $enable_pgsql" +echo "Oracle (OCI) support enabled: $enable_oracle" echo "SNMP support enabled: $enable_snmp" echo "Mail support enabled: $enable_mail" echo "RELP support enabled: $enable_relp" diff --git a/plugins/omoracle/Makefile.am b/plugins/omoracle/Makefile.am index 6b75218f..11257fb2 100644 --- a/plugins/omoracle/Makefile.am +++ b/plugins/omoracle/Makefile.am @@ -5,4 +5,4 @@ omoracle_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(ORACLE_CFLAGS) omoracle_la_LDFLAGS = -module -avoid-version omoracle_la_LIBADD = $(ORACLE_LIBS) -EXTRA_DIST = +#EXTRA_DIST = -- cgit From 57dcc5c6c96aa72b24c5f8c9952f789d08943383 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:26 +0100 Subject: Add all other blocks (macros) needed to make this module work. At this stage they are all empty, but at least it should be possible to instantiate the module and perform some basic tests. Fix some compilation warnings --- plugins/omoracle/omoracle.c | 61 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 696cd908..6077594f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -61,7 +61,7 @@ typedef struct _instanceData { static int oci_errors(void* handle, ub4 htype, sword status) { sb4 errcode; - char buf[MAX_BUFSIZE]; + unsigned char buf[MAX_BUFSIZE]; switch (status) { case OCI_SUCCESS: @@ -76,7 +76,8 @@ static int oci_errors(void* handle, ub4 htype, sword status) case OCI_ERROR: printf ("OCI GENERAL ERROR\n"); if (handle) { - OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); + OCIErrorGet(handle, 1, NULL, &errcode, buf, + sizeof buf, htype); printf ("Error message: %s", buf); } else printf ("NULL handle\n" @@ -100,19 +101,19 @@ static int oci_errors(void* handle, ub4 htype, sword status) BEGINcreateInstance CODESTARTcreateInstance CHECKENV(pData->environment, - OCIEnvCreate(&(pData->environment), OCI_DEFAULT, + OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->error), + OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->server), + OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->service), + OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->authinfo), + OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); finalize_it: ENDcreateInstance @@ -131,9 +132,43 @@ RETiRet; ENDfreeInstance -/* BEGINmodInit() */ -/* CODESTARTmodInit */ -/* *ipIFVersProvided = CURR_MOD_IF_VERSION; */ -/* CODEmodInit_QueryRegCFSLineHdlr */ -/* CHKiRet(objUse(errmsg, CORE_COMPONENT)); */ -/* ENDmodInit */ + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +/* Right now, this module is compatible with nothing. */ +ENDisCompatibleWithFeature + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINdoAction +CODESTARTdoAction +ENDdoAction + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +ENDmodInit -- cgit From 0676277119bd39562a0b8c0de6d2fee23e1deb11 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:27 +0100 Subject: Add handlers on modInit. This avoids crashes on initialization. --- plugins/omoracle/omoracle.c | 78 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 6077594f..517b7173 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -68,29 +68,29 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_SUCCESS; break; case OCI_SUCCESS_WITH_INFO: - printf ("OCI SUCCESS - With info\n"); + dbgprintf ("OCI SUCCESS - With info\n"); break; case OCI_NEED_DATA: - printf ("OCI NEEDS MORE DATA\n"); + dbgprintf ("OCI NEEDS MORE DATA\n"); break; case OCI_ERROR: - printf ("OCI GENERAL ERROR\n"); + dbgprintf ("OCI GENERAL ERROR\n"); if (handle) { OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); - printf ("Error message: %s", buf); + dbgprintf ("Error message: %s", buf); } else - printf ("NULL handle\n" + dbgprintf ("NULL handle\n" "Unable to extract further information"); break; case OCI_INVALID_HANDLE: - printf ("OCI INVALID HANDLE\n"); + dbgprintf ("OCI INVALID HANDLE\n"); break; case OCI_STILL_EXECUTING: - printf ("Still executing...\n"); + dbgprintf ("Still executing...\n"); break; case OCI_CONTINUE: - printf ("OCI CONTINUE\n"); + dbgprintf ("OCI CONTINUE\n"); break; } return OCI_ERROR; @@ -100,21 +100,30 @@ static int oci_errors(void* handle, ub4 htype, sword status) /* Resource allocation */ BEGINcreateInstance CODESTARTcreateInstance + +ASSERT(pData != NULL); + +dbgprintf ("***** OMORACLE ***** Creating instance\n"); CHECKENV(pData->environment, OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created environment\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created error\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created server\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created service\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created authinfo\n"); finalize_it: ENDcreateInstance @@ -122,11 +131,18 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance +dbgprintf ("***** OMORACLE ***** Destroying instance\n"); + OCIHandleFree(pData->environment, OCI_HTYPE_ENV); +dbgprintf ("***** OMORACLE ***** Destroyed environment\n"); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); +dbgprintf ("***** OMORACLE ***** Destroyed error\n"); OCIHandleFree(pData->server, OCI_HTYPE_SERVER); +dbgprintf ("***** OMORACLE ***** Destroyed server\n"); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); +dbgprintf ("***** OMORACLE ***** Destroyed service\n"); OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); +dbgprintf ("***** OMORACLE ***** Destroyed authinfo\n"); RETiRet; @@ -135,40 +151,84 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + +dbgprintf ("***** OMORACLE ***** At tryResume\n"); ENDtryResume BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ +dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); +iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); + +if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); +} + +CHKiRet(createInstance(&pData)); + +p += sizeof ":omoracle:" - 1; +if (*p != ';') { + dbgprintf ("***** OMORACLE ***** Wrong char: %c\n", *p); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); +} +p++; + +CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, + OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + +dbgprintf ("***** OMORACLE ***** Salido\n"); + + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction CODESTARTdoAction +dbgprintf ("***** OMORACLE ***** At doAction\n"); ENDdoAction BEGINmodExit CODESTARTmodExit +dbgprintf ("***** OMORACLE ***** At modExit\n"); ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo +dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); + ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); + ENDqueryEtryPt +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, + void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr - CHKiRet(objUse(errmsg, CORE_COMPONENT)); +CHKiRet(objUse(errmsg, CORE_COMPONENT)); +/* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ +CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, + eCmdHdlrCustomHandler, resetConfigVariables, + NULL, STD_LOADABLE_MODULE_ID)); + +dbgprintf ("***** OMORACLE ***** At modInit\n"); ENDmodInit -- cgit From 3dabb2976a2f259ba1f3bd9823ddd2860edc293d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:28 +0100 Subject: Add the ability to connect to the DB based on the config line. It will read and parse the config line (this code is not yet rock-solid) and connect to the database at initialization time. I also cleaned some debug messages that are not needed anymore. --- plugins/omoracle/omoracle.c | 58 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 517b7173..ab9e7e36 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -103,27 +103,22 @@ CODESTARTcreateInstance ASSERT(pData != NULL); -dbgprintf ("***** OMORACLE ***** Creating instance\n"); CHECKENV(pData->environment, OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created environment\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created error\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created server\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created service\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created authinfo\n"); + finalize_it: ENDcreateInstance @@ -155,6 +150,26 @@ CODESTARTtryResume dbgprintf ("***** OMORACLE ***** At tryResume\n"); ENDtryResume +static rsRetVal startSession(instanceData* pData, char* connection, char* user, + char * password) +{ + DEFiRet; + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, user, + strlen(user), OCI_ATTR_USERNAME, pData->error)); + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, password, + strlen(password), OCI_ATTR_PASSWORD, pData->error)); + CHECKERR(pData->error, + OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, connection, + strlen(connection), NULL, 0, NULL, NULL, NULL, + OCI_DEFAULT)); +finalize_it: + RETiRet; +} + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ @@ -163,6 +178,12 @@ iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct + +char connection_string[MAXHOSTNAMELEN]; +char user[_DB_MAXUNAMELEN]; +char pwd[_DB_MAXPWDLEN]; + + CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -170,20 +191,25 @@ if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); } -CHKiRet(createInstance(&pData)); p += sizeof ":omoracle:" - 1; -if (*p != ';') { - dbgprintf ("***** OMORACLE ***** Wrong char: %c\n", *p); + +if (*p == '\0' || *p == ',') { + dbgprintf ("Wrong char processing module arguments: %c\n", *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } -p++; +CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); +CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); +CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); +p--; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); +CHKiRet(createInstance(&pData)); +CHKiRet(startSession(pData, connection_string, user, pwd)); -dbgprintf ("***** OMORACLE ***** Salido\n"); - +dbgprintf ("omoracle module got all its resources allocated " + "and connected to the DB\n"); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -223,12 +249,16 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; +char dbname[MAX_BUFSIZE]; CODEmodInit_QueryRegCFSLineHdlr +dbgprintf ("***** OMORACLE ***** At modInit\n"); CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); - -dbgprintf ("***** OMORACLE ***** At modInit\n"); +dbgprintf ("***** OMORACLE ***** dbname before = %s\n", dbname); +CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, + NULL, dbname, STD_LOADABLE_MODULE_ID)); +dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); ENDmodInit -- cgit From 47b334bef1ea9e72bc41279c42228724f6e141f6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:29 +0100 Subject: Add the ability to actually run statements. It now runs SQL statements given as templates. In this case, the template is given on the configuration file and the core passes the SQL statement correctly formatted to doAction. I still need to decide how to structure this for having prepared statements (prepare them at parseSelector time) and then make doAction to only bind arguments and execute. It commits after each statement, which is awfully slow but good enough for the moment. Next step after that is have a buffer of arguments, and make doAction store new data as it arrives, then run the statement only when the buffer is almost full. Or something like that. --- plugins/omoracle/omoracle.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ab9e7e36..c1900a73 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -4,6 +4,13 @@ database. It uses Oracle Call Interface, a propietary module provided by Oracle. + Config lines to be used are of this form: + + :omoracle:dbstring,user,password;StatementTemplate + + All fields are mandatory. The dbstring can be an Oracle easystring + or a DB name, as present in the tnsnames.ora file. + Author: Luis Fernando Muñoz Mejías @@ -39,7 +46,6 @@ typedef struct _instanceData { OCIEnv* environment; OCISession* session; OCIError* error; - OCIServer* server; OCIStmt* statement; OCISvcCtx* service; OCIAuthInfo* authinfo; @@ -109,15 +115,12 @@ CHECKENV(pData->environment, CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->server), - OCI_HTYPE_SERVER, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->service), - OCI_HTYPE_SVCCTX, 0, NULL)); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->statement), + OCI_HTYPE_STMT, 0, NULL)); finalize_it: ENDcreateInstance @@ -126,18 +129,13 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance -dbgprintf ("***** OMORACLE ***** Destroying instance\n"); - +OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->environment, OCI_HTYPE_ENV); -dbgprintf ("***** OMORACLE ***** Destroyed environment\n"); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); -dbgprintf ("***** OMORACLE ***** Destroyed error\n"); -OCIHandleFree(pData->server, OCI_HTYPE_SERVER); -dbgprintf ("***** OMORACLE ***** Destroyed server\n"); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); -dbgprintf ("***** OMORACLE ***** Destroyed service\n"); OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); -dbgprintf ("***** OMORACLE ***** Destroyed authinfo\n"); +OCIHandleFree(pData->statement, OCI_HTYPE_STMT); +dbgprintf ("omoracle freed all its resources\n"); RETiRet; @@ -216,12 +214,21 @@ ENDparseSelectorAct BEGINdoAction CODESTARTdoAction -dbgprintf ("***** OMORACLE ***** At doAction\n"); + dbgprintf("omoracle attempting to execute statement %s\n", *ppString); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, pData->error, *ppString, + strlen(*ppString), OCI_NTV_SYNTAX, + OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, pData->statement, pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); +finalize_it: ENDdoAction BEGINmodExit CODESTARTmodExit -dbgprintf ("***** OMORACLE ***** At modExit\n"); ENDmodExit BEGINdbgPrintInstInfo @@ -262,3 +269,4 @@ CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, NULL, dbname, STD_LOADABLE_MODULE_ID)); dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); ENDmodInit + -- cgit From b6123427cf962e70836c07d1c5c2cf39978673b8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:30 +0100 Subject: Add proper indentation (despite Emacs) and support for retrying. Emacs doesn't allow for proper indentation with rsyslog's macros (no curly brackets, so it doesn't know where functions start), so I had to manually add such indentation. Add support for retrying actions, namely, disconnect from the DB, re-connecting and re-executing the last prepared statement. Needs to be tested. --- plugins/omoracle/omoracle.c | 168 ++++++++++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 78 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index c1900a73..aa506dca 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -50,6 +50,7 @@ typedef struct _instanceData { OCISvcCtx* service; OCIAuthInfo* authinfo; OCIBind* binding; + char* connection; } instanceData; /** Generic function for handling errors from OCI. @@ -74,29 +75,30 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_SUCCESS; break; case OCI_SUCCESS_WITH_INFO: - dbgprintf ("OCI SUCCESS - With info\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI SUCCESS - With info\n"); break; case OCI_NEED_DATA: - dbgprintf ("OCI NEEDS MORE DATA\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI NEEDS MORE DATA\n"); break; case OCI_ERROR: dbgprintf ("OCI GENERAL ERROR\n"); if (handle) { OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); - dbgprintf ("Error message: %s", buf); + errmsg.LogError(0, NO_ERRCODE, "Error message: %s", buf); } else - dbgprintf ("NULL handle\n" - "Unable to extract further information"); + errmsg.LogError(0, NO_ERRCODE, "NULL handle\n" + "Unable to extract further " + "information"); break; case OCI_INVALID_HANDLE: - dbgprintf ("OCI INVALID HANDLE\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI INVALID HANDLE\n"); break; case OCI_STILL_EXECUTING: - dbgprintf ("Still executing...\n"); + errmsg.LogError(0, NO_ERRCODE, "Still executing...\n"); break; case OCI_CONTINUE: - dbgprintf ("OCI CONTINUE\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI CONTINUE\n"); break; } return OCI_ERROR; @@ -107,45 +109,56 @@ static int oci_errors(void* handle, ub4 htype, sword status) BEGINcreateInstance CODESTARTcreateInstance -ASSERT(pData != NULL); - -CHECKENV(pData->environment, - OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, - NULL, NULL, NULL, NULL, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->error), - OCI_HTYPE_ERROR, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), - OCI_HTYPE_AUTHINFO, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->statement), - OCI_HTYPE_STMT, 0, NULL)); + ASSERT(pData != NULL); + + CHECKENV(pData->environment, + OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, + NULL, NULL, NULL, NULL, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->error), + OCI_HTYPE_ERROR, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), + OCI_HTYPE_AUTHINFO, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->statement), + OCI_HTYPE_STMT, 0, NULL)); finalize_it: ENDcreateInstance -/** Free any resources allocated by createInstance. */ +/** Close the session and free anything allocated by + createInstance. */ BEGINfreeInstance CODESTARTfreeInstance -OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); -OCIHandleFree(pData->environment, OCI_HTYPE_ENV); -OCIHandleFree(pData->error, OCI_HTYPE_ERROR); -OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); -OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); -OCIHandleFree(pData->statement, OCI_HTYPE_STMT); -dbgprintf ("omoracle freed all its resources\n"); - -RETiRet; + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->environment, OCI_HTYPE_ENV); + OCIHandleFree(pData->error, OCI_HTYPE_ERROR); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); + OCIHandleFree(pData->statement, OCI_HTYPE_STMT); + free(pData->connection); + dbgprintf ("omoracle freed all its resources\n"); + RETiRet; ENDfreeInstance - BEGINtryResume CODESTARTtryResume + dbgprintf("Attempting to restart the last action\n"); + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, + pData->connection, + strlen(pData->connection), NULL, 0, + NULL, NULL, NULL, OCI_DEFAULT)); + CHECKERR(pData->error, OCIStmtExecute(pData->service, pData->statement, + pData->error, 1, 0, NULL, NULL, + OCI_DEFAULT)); -dbgprintf ("***** OMORACLE ***** At tryResume\n"); +finalize_it: ENDtryResume static rsRetVal startSession(instanceData* pData, char* connection, char* user, @@ -170,45 +183,50 @@ finalize_it: BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature -/* Right now, this module is compatible with nothing. */ -dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); -iRet = RS_RET_INCOMPATIBLE; + /* Right now, this module is compatible with nothing. */ + dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); + iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct -char connection_string[MAXHOSTNAMELEN]; -char user[_DB_MAXUNAMELEN]; -char pwd[_DB_MAXPWDLEN]; - + char user[_DB_MAXUNAMELEN]; + char pwd[_DB_MAXPWDLEN]; + char connection_string[MAXHOSTNAMELEN]; CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); -if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { - ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); -} - - -p += sizeof ":omoracle:" - 1; - -if (*p == '\0' || *p == ',') { - dbgprintf ("Wrong char processing module arguments: %c\n", *p); - ABORT_FINALIZE(RS_RET_INVALID_PARAMS); -} + if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } -CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); -CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); -CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); -p--; -CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, - OMSR_RQD_TPL_OPT_SQL, " StdFmt")); -CHKiRet(createInstance(&pData)); -CHKiRet(startSession(pData, connection_string, user, pwd)); + p += sizeof ":omoracle:" - 1; -dbgprintf ("omoracle module got all its resources allocated " - "and connected to the DB\n"); + if (*p == '\0' || *p == ',') { + errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); + CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); + CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); + p--; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, + OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + CHKiRet(createInstance(&pData)); + pData->connection = strdup(connection_string); + if (pData->connection == NULL) { + iRet = RS_RET_OUT_OF_MEMORY; + goto finalize_it; + } + CHKiRet(startSession(pData, connection_string, user, pwd)); + + dbgprintf ("omoracle module got all its resources allocated " + "and connected to the DB\n"); + + memset(user, 0, sizeof user); + memset(pwd, 0, sizeof pwd); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -225,6 +243,9 @@ CODESTARTdoAction CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); finalize_it: + dbgprintf ("omoracle %s at executing statement %s\n", + iRet?"did not succeed":"succeeded", *ppString); +/* Clean credentials to avoid leakage in case of core dump. */ ENDdoAction BEGINmodExit @@ -233,16 +254,14 @@ ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo -dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); - + dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES -dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); - + dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); ENDqueryEtryPt static rsRetVal @@ -256,17 +275,10 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; -char dbname[MAX_BUFSIZE]; CODEmodInit_QueryRegCFSLineHdlr -dbgprintf ("***** OMORACLE ***** At modInit\n"); -CHKiRet(objUse(errmsg, CORE_COMPONENT)); -/* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ -CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, - eCmdHdlrCustomHandler, resetConfigVariables, - NULL, STD_LOADABLE_MODULE_ID)); -dbgprintf ("***** OMORACLE ***** dbname before = %s\n", dbname); -CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, - NULL, dbname, STD_LOADABLE_MODULE_ID)); -dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ + CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, + eCmdHdlrCustomHandler, resetConfigVariables, + NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit - -- cgit From 0289fb7f5c63ee1563eb829b5b1a0fcf3dfa279a Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:31 +0100 Subject: Remove useless dbgprintf and add documentation. --- plugins/omoracle/omoracle.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index aa506dca..4cf4c724 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -43,13 +43,21 @@ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) typedef struct _instanceData { + /* Environment handler, the base for any OCI work. */ OCIEnv* environment; + /* Session handler, the actual DB connection object. */ OCISession* session; + /* Error handler for OCI calls. */ OCIError* error; + /* Prepared statement. */ OCIStmt* statement; + /* Service handler. */ OCISvcCtx* service; + /* Credentials object for the connection. */ OCIAuthInfo* authinfo; + /* Binding parameters, currently unused */ OCIBind* binding; + /* Connection string, kept here for possible retries. */ char* connection; } instanceData; @@ -177,6 +185,8 @@ static rsRetVal startSession(instanceData* pData, char* connection, char* user, strlen(connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); finalize_it: + if (iRet != RS_RET_OK) + errmsg.LogError(0, NO_ERRCODE, "Unable to start Oracle session\n"); RETiRet; } @@ -254,14 +264,12 @@ ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES - dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); ENDqueryEtryPt static rsRetVal -- cgit From c54de8212fe434080983b70d9ffa200d52f68ccd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 25 Mar 2009 19:41:17 +0100 Subject: added some (hopefully helpful) comments on the calling IF --- plugins/omoracle/omoracle.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 4cf4c724..eba45c5b 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -154,6 +154,26 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + /* Here usually only a reconnect is done. The rsyslog core will call + * this entry point from time to time when the action suspended itself. + * Note that the rsyslog core expects that if the plugin suspended itself + * the action was not carried out during that invocation. Thus, rsyslog + * will call the action with *the same* data item again AFTER a resume + * was successful. As such, tryResume should NOT write the failed data + * item. If it needs to for some reason, it must delete the item again, + * otherwise, it will get duplicated. + * This handling inside the rsyslog core is important to be able to + * preserve data over rsyslog restarts. With it, the core can ensure that + * it queues all not-yet-processed messages without the plugin needing + * to take care about that. + * So in essence, it is recommended that just a reconnet is tried, but + * the last action not restarted. Note that it is not a real problem + * (but causes a slight performance degradation) if tryResume returns + * successfully but the next call to doAction() immediately returns + * RS_RET_SUSPENDED. So it is OK to do the actual restart inside doAction(). + * ... of course I don't know why Oracle might need a full restart... + * rgerhards, 2009-03-26 + */ dbgprintf("Attempting to restart the last action\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); @@ -213,6 +233,13 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); p += sizeof ":omoracle:" - 1; + /* while this parameter parsing is convenient and works perfectly, + * it is suggested that parameters are only specified via $Action... config + * statement (as done in omlibdbi). The reason is that this may greatly + * ease the transition when we have the full config script language. However, + * this approach here is guranteed to continue to work in the future. + * rgerhards, 2009-03-26 + */ if (*p == '\0' || *p == ',') { errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); -- cgit From 59d4a52c280c00bccde4be0321bb09665cc11d29 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 16:31:41 +0200 Subject: initial work on omprog, an output module to send messages to another program --- Makefile.am | 6 +- configure.ac | 16 ++++ doc/features.html | 1 - plugins/omdtn/Makefile.am | 8 -- plugins/omdtn/omdtn.c | 130 ----------------------------- plugins/omprog/Makefile.am | 8 ++ plugins/omprog/omprog.c | 204 +++++++++++++++++++++++++++++++++++++++++++++ runtime/conf.c | 1 - runtime/rsyslog.h | 2 + 9 files changed, 235 insertions(+), 141 deletions(-) delete mode 100644 plugins/omdtn/Makefile.am delete mode 100644 plugins/omdtn/omdtn.c create mode 100644 plugins/omprog/Makefile.am create mode 100644 plugins/omprog/omprog.c diff --git a/Makefile.am b/Makefile.am index 97f4aed3..02444d6f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -108,6 +108,10 @@ if ENABLE_MAIL SUBDIRS += plugins/ommail endif +if ENABLE_OMPROG +SUBDIRS += plugins/omprog +endif + if ENABLE_RFC3195 SUBDIRS += plugins/im3195 endif @@ -122,5 +126,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index de328f83..e9e68cb4 100644 --- a/configure.ac +++ b/configure.ac @@ -668,6 +668,20 @@ AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to serach for imtemplate to find everything! +# settings for the omprog output module +AC_ARG_ENABLE(omprog, + [AS_HELP_STRING([--enable-omprog],[Compiles omprog template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omprog="yes" ;; + no) enable_omprog="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omprog) ;; + esac], + [enable_omprog=no] +) +AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) +# end of omprog + + # settings for the template output module; copy and modify this code # if you intend to add your own module. Be sure to replace omtemplate # by the actual name of your module. @@ -720,6 +734,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ plugins/omtemplate/Makefile \ + plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ @@ -752,6 +767,7 @@ echo "imdiag enabled: $enable_imdiag" echo "file input module enabled: $enable_imfile" echo "input template module will be compiled: $enable_imtemplate" echo "output template module will be compiled: $enable_omtemplate" +echo "omprog module will be compiled: $enable_omprog" echo "omstdout module will be compiled: $enable_omstdout" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" diff --git a/doc/features.html b/doc/features.html index 501f3304..336b31cc 100644 --- a/doc/features.html +++ b/doc/features.html @@ -124,7 +124,6 @@ community. Plus, it can be financially attractive: just think about how much les be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit of being recognised as a sponsor may even drive new customers to your business!
    -
  • Finalize the DTN "planetary Internet" space ship mode output plugin
  • port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge
  • pcre filtering - maybe (depending on feedback)  - diff --git a/plugins/omdtn/Makefile.am b/plugins/omdtn/Makefile.am deleted file mode 100644 index afb57476..00000000 --- a/plugins/omdtn/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -pkglib_LTLIBRARIES = omdtn.la - -omdtn_la_SOURCES = omdtn.c -omdtn_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -omdtn_la_LDFLAGS = -module -avoid-version -omdtn_la_LIBADD = - -EXTRA_DIST = diff --git a/plugins/omdtn/omdtn.c b/plugins/omdtn/omdtn.c deleted file mode 100644 index 761bde79..00000000 --- a/plugins/omdtn/omdtn.c +++ /dev/null @@ -1,130 +0,0 @@ -/* omdtn.c - * This is the plugin for rsyslog use in the interplanetary Internet, - * especially useful for rsyslog in space ships of all kinds. - * The core idea was introduced in early 2009 and considered - * doable. - * - * Note that this has not yet been tested for robustness but needs - * to prior to placing it on top of a rocket. - * - * NOTE: read comments in module-template.h for more specifics! - * - * File begun on 2009-04-01 by RGerhards - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include "rsyslog.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "dirty.h" -#include "syslogd-types.h" -#include "srUtils.h" -#include "template.h" -#include "module-template.h" -#include "errmsg.h" -#include "cfsysline.h" - -MODULE_TYPE_OUTPUT - -/* internal structures - */ -DEF_OMOD_STATIC_DATA - -typedef struct _instanceData { -} instanceData; - -BEGINcreateInstance -CODESTARTcreateInstance -ENDcreateInstance - - -BEGINisCompatibleWithFeature -CODESTARTisCompatibleWithFeature - if(eFeat == sFEATURERepeatedMsgReduction) - iRet = RS_RET_OK; -ENDisCompatibleWithFeature - - -BEGINfreeInstance -CODESTARTfreeInstance -ENDfreeInstance - - -BEGINdbgPrintInstInfo -CODESTARTdbgPrintInstInfo -ENDdbgPrintInstInfo - - -BEGINtryResume -CODESTARTtryResume -ENDtryResume - -BEGINdoAction -CODESTARTdoAction - write(1, (char*)ppString[0], strlen((char*)ppString[0])); -ENDdoAction - - -BEGINparseSelectorAct -CODESTARTparseSelectorAct -CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* first check if this config line is actually for us */ - if(strncmp((char*) p, ":omstdout:", sizeof(":omstdout:") - 1)) { - ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); - } - - /* ok, if we reach this point, we have something for us */ - p += sizeof(":omstdout:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ - CHKiRet(createInstance(&pData)); - - /* check if a non-standard template is to be applied */ - if(*(p-1) == ';') - --p; - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); -CODE_STD_FINALIZERparseSelectorAct -ENDparseSelectorAct - - -BEGINmodExit -CODESTARTmodExit -ENDmodExit - - -BEGINqueryEtryPt -CODESTARTqueryEtryPt -CODEqueryEtryPt_STD_OMOD_QUERIES -ENDqueryEtryPt - - -BEGINmodInit() -CODESTARTmodInit - *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ -CODEmodInit_QueryRegCFSLineHdlr -ENDmodInit - -/* vi:set ai: - */ diff --git a/plugins/omprog/Makefile.am b/plugins/omprog/Makefile.am new file mode 100644 index 00000000..63fe09b8 --- /dev/null +++ b/plugins/omprog/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omprog.la + +omprog_la_SOURCES = omprog.c +omprog_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omprog_la_LDFLAGS = -module -avoid-version +omprog_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c new file mode 100644 index 00000000..0cdacf78 --- /dev/null +++ b/plugins/omprog/omprog.c @@ -0,0 +1,204 @@ +/* omprog.c + * This output plugin enables rsyslog to execute a program and + * feed it the message stream as standard input. + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-04-01 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +typedef struct _instanceData { + uchar *szBinary; /* name of binary to call */ + pid_t pid; /* pid of currently running process */ + int fdPipe; /* file descriptor to write to */ + int bIsRunning; /* is binary currently running? 0-no, 1-yes */ +} instanceData; + +/* config settings */ +static uchar *szBinary = NULL; /* name of binary to call */ + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +/* creates a pipe and starts program, uses pipe as stdin for program. + * rgerhards, 2009-04-01 + */ +static rsRetVal +openPipe(instanceData *pData) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = { NULL }; + char *newenviron[] = { NULL }; + DEFiRet; + + assert(pData != NULL); + + if(pipe(pipefd) == -1) { + ABORT_FINALIZE(RS_RET_ERR_CREAT_PIPE); + } + + cpid = fork(); + if(cpid == -1) { + ABORT_FINALIZE(RS_RET_ERR_FORK); + } + + if(cpid == 0) { + /* we are now the child, just set the right selectors and + * exec the binary. If that fails, there is not much we can do. + */ + fclose(stdin); + dup(pipefd[0]); + close(pipefd[1]); + //fclose(stdout); +fprintf(stderr, "Program to exec '%s', fdPipe: %d\n", pData->szBinary, pipefd[0]); + execve((char*)pData->szBinary, newargv, newenviron); + } + + pData->fdPipe = pipefd[1]; + pData->pid = cpid; + close(pipefd[0]); + pData->bIsRunning = 1; +finalize_it: + RETiRet; +} + + +BEGINdoAction +CODESTARTdoAction + if(pData->bIsRunning == 0) { + openPipe(pData); + } + + write(pData->fdPipe, (char*)ppString[0], strlen((char*)ppString[0])); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omprog:", sizeof(":omprog:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omprog:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + CHKmalloc(pData->szBinary = (uchar*) strdup((char*)szBinary)); + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit + if(szBinary != NULL) { + free(szBinary); + szBinary = NULL; + } +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + + if(szBinary != NULL) { + free(szBinary); + szBinary = NULL; + } + + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomprogbinary", 0, eCmdHdlrGetWord, NULL, &szBinary, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit + +/* vi:set ai: + */ diff --git a/runtime/conf.c b/runtime/conf.c index ede15cc7..27ab8bb4 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -796,7 +796,6 @@ dbgprintf("calling expression parser, pp %p ('%s')\n", *pline, *pline); /* debug support - print vmprg after construction (uncomment to use) */ /* vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); */ - vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); /* we now need to skip whitespace to the action part, else we confuse * the legacy rsyslog conf parser. -- rgerhards, 2008-02-25 diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 899f5e13..2aa1868b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -258,6 +258,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ + RS_RET_ERR_CREAT_PIPE = -2114, /**< error during pipe creation */ + RS_RET_ERR_FORK = -2115, /**< error during fork() */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From 01f2c7a7a3ee394028eb36d9709490cbc26c7369 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 18:17:20 +0200 Subject: improved omprog, now ready for first practical testing --- ChangeLog | 3 + plugins/omprog/omprog.c | 169 +++++++++++++++++++++++++++++++++++++++++++++--- runtime/rsyslog.h | 1 + 3 files changed, 165 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 23a2e3e0..77ebdf83 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- new feature: new output plugin omprog, which permits to start program + and feed it (via its stdin) with syslog messages. If the program + terminates, it is restarted. - bugfix: potential abort with DA queue after high watermark is reached There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 0cdacf78..2a078a6d 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "dirty.h" #include "syslogd-types.h" #include "srUtils.h" @@ -48,6 +49,7 @@ MODULE_TYPE_OUTPUT /* internal structures */ DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) typedef struct _instanceData { uchar *szBinary; /* name of binary to call */ @@ -73,6 +75,8 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance + if(pData->szBinary != NULL) + free(pData->szBinary); ENDfreeInstance @@ -85,6 +89,52 @@ BEGINtryResume CODESTARTtryResume ENDtryResume + +/* execute the child process (must be called in child context + * after fork). + */ + +static void execBinary(instanceData *pData, int fdStdin) +{ + int i; + struct sigaction sigAct; + char *newargv[] = { NULL }; + char *newenviron[] = { NULL }; + + assert(pData != NULL); + + fclose(stdin); + dup(fdStdin); + //fclose(stdout); + + /* we close all file handles as we fork soon + * Is there a better way to do this? - mail me! rgerhards@adiscon.com + */ +# ifndef VALGRIND /* we can not use this with valgrind - too many errors... */ + for(i = 3 ; i <= 65535 ; ++i) + close(i); +# endif + + /* reset signal handlers to default */ + memset(&sigAct, 0, sizeof(sigAct)); + sigfillset(&sigAct.sa_mask); + sigAct.sa_handler = SIG_DFL; + for(i = 1 ; i < NSIG ; ++i) + sigaction(i, &sigAct, NULL); + + alarm(0); + + /* finally exec child */ + execve((char*)pData->szBinary, newargv, newenviron); + /* switch to? + execlp((char*)program, (char*) program, (char*)arg, NULL); + */ + + /* we should never reach this point, but if we do, we terminate */ + exit(1); +} + + /* creates a pipe and starts program, uses pipe as stdin for program. * rgerhards, 2009-04-01 */ @@ -93,8 +143,6 @@ openPipe(instanceData *pData) { int pipefd[2]; pid_t cpid; - char *newargv[] = { NULL }; - char *newenviron[] = { NULL }; DEFiRet; assert(pData != NULL); @@ -103,6 +151,10 @@ openPipe(instanceData *pData) ABORT_FINALIZE(RS_RET_ERR_CREAT_PIPE); } + DBGPRINTF("executing program '%s'\n", pData->szBinary); + + /* NO OUTPUT AFTER FORK! */ + cpid = fork(); if(cpid == -1) { ABORT_FINALIZE(RS_RET_ERR_FORK); @@ -112,14 +164,12 @@ openPipe(instanceData *pData) /* we are now the child, just set the right selectors and * exec the binary. If that fails, there is not much we can do. */ - fclose(stdin); - dup(pipefd[0]); close(pipefd[1]); - //fclose(stdout); -fprintf(stderr, "Program to exec '%s', fdPipe: %d\n", pData->szBinary, pipefd[0]); - execve((char*)pData->szBinary, newargv, newenviron); + execBinary(pData, pipefd[0]); + /*NO CODE HERE - WILL NEVER BE REACHED!*/ } + DBGPRINTF("child has pid %d\n", cpid); pData->fdPipe = pipefd[1]; pData->pid = cpid; close(pipefd[0]); @@ -129,13 +179,113 @@ finalize_it: } +/* clean up after a terminated child + */ +static inline rsRetVal +cleanup(instanceData *pData) +{ + int status; + int ret; + char errStr[1024]; + DEFiRet; + + assert(pData != NULL); + assert(pData->bIsRunning == 1); +RUNLOG_VAR("%d", pData->pid); + ret = waitpid(pData->pid, &status, 0); + if(ret != pData->pid) { + /* if waitpid() fails, we can not do much - try to ignore it... */ + DBGPRINTF("waitpid() returned state %d[%s], future malfunction may happen\n", ret, + rs_strerror_r(errno, errStr, sizeof(errStr))); + } else { + /* check if we should print out some diagnostic information */ + DBGPRINTF("waitpid status return for program '%s': %2.2x\n", + pData->szBinary, status); + if(WIFEXITED(status)) { + errmsg.LogError(0, NO_ERRCODE, "program '%s' exited normally, state %d", + pData->szBinary, WEXITSTATUS(status)); + } else if(WIFSIGNALED(status)) { + errmsg.LogError(0, NO_ERRCODE, "program '%s' terminated by signal %d.", + pData->szBinary, WTERMSIG(status)); + } + } + + pData->bIsRunning = 0; + RETiRet; +} + + +/* try to restart the binary when it has stopped. + */ +static inline rsRetVal +tryRestart(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + assert(pData->bIsRunning == 0); + + iRet = openPipe(pData); + RETiRet; +} + + +/* write to pipe + * note that we do not try to run block-free. If the users fears something + * may block (and this not be acceptable), the action should be run on its + * own action queue. + */ +static rsRetVal +writePipe(instanceData *pData, uchar *szMsg) +{ + int lenWritten; + int lenWrite; + int writeOffset; + char errStr[1024]; + DEFiRet; + + assert(pData != NULL); + + lenWrite = strlen((char*)szMsg); + writeOffset = 0; + + do + { + lenWritten = write(pData->fdPipe, ((char*)szMsg)+writeOffset, lenWrite); + if(lenWritten == -1) { + switch(errno) { + case EPIPE: + DBGPRINTF("Program '%s' terminated, trying to restart\n", + pData->szBinary); + CHKiRet(cleanup(pData)); + CHKiRet(tryRestart(pData)); + break; + default: + DBGPRINTF("error %d writing to pipe: %s\n", errno, + rs_strerror_r(errno, errStr, sizeof(errStr))); + ABORT_FINALIZE(RS_RET_ERR_WRITE_PIPE); + break; + } + } else { + writeOffset += lenWritten; + } + } while(lenWritten != lenWrite); + + +finalize_it: + RETiRet; +} + + BEGINdoAction CODESTARTdoAction if(pData->bIsRunning == 0) { openPipe(pData); } + + iRet = writePipe(pData, ppString[0]); - write(pData->fdPipe, (char*)ppString[0], strlen((char*)ppString[0])); + if(iRet != RS_RET_OK) + iRet = RS_RET_SUSPENDED; ENDdoAction @@ -166,6 +316,8 @@ CODESTARTmodExit free(szBinary); szBinary = NULL; } + CHKiRet(objRelease(errmsg, CORE_COMPONENT)); +finalize_it: ENDmodExit @@ -195,6 +347,7 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomprogbinary", 0, eCmdHdlrGetWord, NULL, &szBinary, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 2aa1868b..ac9b726f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -260,6 +260,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_ERR_CREAT_PIPE = -2114, /**< error during pipe creation */ RS_RET_ERR_FORK = -2115, /**< error during fork() */ + RS_RET_ERR_WRITE_PIPE = -2116, /**< error writing to pipe */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From d702d3f6ff1540691aae29012dd142212e0eb04c Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Mon, 30 Mar 2009 10:39:36 +0200 Subject: Make tryResume not to retry the last action, but just to reconnect. The core will call the action if tryResume succeeds, no need to make it from here. --- plugins/omoracle/omoracle.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index eba45c5b..29ec2303 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -174,7 +174,7 @@ CODESTARTtryResume * ... of course I don't know why Oracle might need a full restart... * rgerhards, 2009-03-26 */ - dbgprintf("Attempting to restart the last action\n"); + dbgprintf("Attempting to reconnect to DB server\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, @@ -182,9 +182,6 @@ CODESTARTtryResume pData->connection, strlen(pData->connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); - CHECKERR(pData->error, OCIStmtExecute(pData->service, pData->statement, - pData->error, 1, 0, NULL, NULL, - OCI_DEFAULT)); finalize_it: ENDtryResume -- cgit From 56bf679723f2821d0f66339b95d847f1e4ddc17b Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Mon, 30 Mar 2009 10:39:37 +0200 Subject: Add a SELinux policy that allows the module to load on RHEL5. --- plugins/omoracle/omoracle.te | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 plugins/omoracle/omoracle.te diff --git a/plugins/omoracle/omoracle.te b/plugins/omoracle/omoracle.te new file mode 100644 index 00000000..81eb6cf1 --- /dev/null +++ b/plugins/omoracle/omoracle.te @@ -0,0 +1,13 @@ + +module omoracle 1.0; + +require { + type syslogd_t; + type port_t; + class process { execstack execmem }; + class tcp_socket name_connect; +} + +#============= syslogd_t ============== +allow syslogd_t port_t:tcp_socket name_connect; +allow syslogd_t self:process { execstack execmem }; -- cgit From 70b8624c86e7d204d7c1ff91d030ee7c69569eb7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 1 Apr 2009 18:12:46 +0200 Subject: Convert the module configuration to $Action... directives. Instead of using the old-style configuration parameters, use $... directives, which lead to simpler code, and also should make user's configurations simpler. Needs some testing. Currently, the supported directives are $OmoracleDB, $OmoracleDBUser and $OmoracleDBPassword. $OmoracleDBStatement and $OmoracleDBBatchSize may follow. --- plugins/omoracle/omoracle.c | 66 +++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 29ec2303..6ec023e2 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -4,10 +4,18 @@ database. It uses Oracle Call Interface, a propietary module provided by Oracle. - Config lines to be used are of this form: + Selector lines to be used are of this form: - :omoracle:dbstring,user,password;StatementTemplate + :omoracle:;TemplateName + The module gets its configuration via rsyslog $... directives, + namely: + + $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db + name as specified by tnsnames.ora) + All fields are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -61,6 +69,14 @@ typedef struct _instanceData { char* connection; } instanceData; +/** Database name, to be filled by the $OmoracleDB directive */ +static char* db_name; +/** Database user name, to be filled by the $OmoracleDBUser + * directive */ +static char* db_user; +/** Database password, to be filled by the $OmoracleDBPassword */ +static char* db_password; + /** Generic function for handling errors from OCI. It will be called only inside CHECKERR and CHECKENV macros. @@ -203,7 +219,8 @@ static rsRetVal startSession(instanceData* pData, char* connection, char* user, OCI_DEFAULT)); finalize_it: if (iRet != RS_RET_OK) - errmsg.LogError(0, NO_ERRCODE, "Unable to start Oracle session\n"); + errmsg.LogError(0, NO_ERRCODE, + "Unable to start Oracle session\n"); RETiRet; } @@ -217,10 +234,6 @@ ENDisCompatibleWithFeature BEGINparseSelectorAct - char user[_DB_MAXUNAMELEN]; - char pwd[_DB_MAXPWDLEN]; - char connection_string[MAXHOSTNAMELEN]; - CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -230,37 +243,26 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); p += sizeof ":omoracle:" - 1; - /* while this parameter parsing is convenient and works perfectly, - * it is suggested that parameters are only specified via $Action... config - * statement (as done in omlibdbi). The reason is that this may greatly - * ease the transition when we have the full config script language. However, - * this approach here is guranteed to continue to work in the future. - * rgerhards, 2009-03-26 - */ if (*p == '\0' || *p == ',') { - errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); + errmsg.LogError(0, NO_ERRCODE, + "Wrong char processing module arguments: %c\n", + *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } - CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); - CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); - CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); - p--; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); CHKiRet(createInstance(&pData)); - pData->connection = strdup(connection_string); + pData->connection = strdup(db_name); if (pData->connection == NULL) { iRet = RS_RET_OUT_OF_MEMORY; goto finalize_it; } - CHKiRet(startSession(pData, connection_string, user, pwd)); + CHKiRet(startSession(pData, db_name, db_user, db_password)); dbgprintf ("omoracle module got all its resources allocated " "and connected to the DB\n"); - memset(user, 0, sizeof user); - memset(pwd, 0, sizeof pwd); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -300,7 +302,16 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { + int n; DEFiRet; + free(db_user); + free(db_name); + if (db_password != NULL) { + n = strlen(db_password); + memset(db_password, n, sizeof *db_password); + free(db_password); + } + db_name = db_user = db_password = NULL; RETiRet; } @@ -313,4 +324,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbuser", 0, + eCmdHdlrGetWord, NULL, &db_user, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbpassword", 0, + eCmdHdlrGetWord, NULL, &db_password, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, + eCmdHdlrGetWord, NULL, &db_name, + STD_LOADABLE_MODULE_ID)); ENDmodInit -- cgit From 8a819d6a0623c0854462dd2dc632700f9a5c576c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 18:32:18 +0200 Subject: some small changes (as suggestion) --- plugins/omoracle/omoracle.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 6ec023e2..ea910d3a 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -253,11 +253,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); CHKiRet(createInstance(&pData)); - pData->connection = strdup(db_name); - if (pData->connection == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto finalize_it; - } + CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); dbgprintf ("omoracle module got all its resources allocated " @@ -304,11 +300,13 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, { int n; DEFiRet; - free(db_user); - free(db_name); + if(db_user != NULL) + free(db_user); + if(db_name != NULL) + free(db_name); if (db_password != NULL) { n = strlen(db_password); - memset(db_password, n, sizeof *db_password); + memset(db_password, 0, n); free(db_password); } db_name = db_user = db_password = NULL; -- cgit From 8e3c5a9ca3732a41fbb8581b13c49acd699820da Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 17:55:04 +0200 Subject: improved internal handling of RainerScript functions - building the necessary plumbing to support more functions with decent runtime performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" --- ChangeLog | 7 ++ runtime/expr.c | 4 +- runtime/rsyslog.h | 4 + runtime/vm.c | 252 +++++++++++++++++++++++++++++++++++++++++++++++++++--- runtime/vm.h | 5 +- runtime/vmop.c | 53 ++++++++++-- runtime/vmop.h | 11 ++- runtime/vmprg.c | 28 +++++- runtime/vmprg.h | 4 +- runtime/vmstk.h | 4 +- tests/3.rstest | 4 +- tools/syslogd.c | 1 + 12 files changed, 345 insertions(+), 32 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8d4191d3..24898dc7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +--------------------------------------------------------------------------- +Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? +- improved internal handling of RainerScript functions, building the + necessary plumbing to support more functions with decent runtime + performance. This is also necessary towards the long-term goal + of loadable library modules. +- added new RainerScript function "tolower" - bugfix: potential abort with DA queue after high watermark is reached There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which diff --git a/runtime/expr.c b/runtime/expr.c index 38ed1c68..e449d1c7 100644 --- a/runtime/expr.c +++ b/runtime/expr.c @@ -142,7 +142,9 @@ terminal(expr_t *pThis, ctok_t *tok) * we have all relevant information) */ CHKiRet(ctok_token.UnlinkVar(pToken, &pVar)); - CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_FUNC_CALL, pVar)); /* add to program */ + CHKiRet(var.ConvToString(pVar)); /* make sure we have a string */ + CHKiRet(vmprg.AddCallOperation(pThis->pVmprg, pVar->val.pStr)); /* add to program */ + CHKiRet(var.Destruct(&pVar)); break; case ctok_MSGVAR: dbgoprint((obj_t*) pThis, "MSGVAR\n"); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 032d8c04..cea457d8 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -84,6 +84,8 @@ typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; +typedef struct vmstk_s vmstk_t; +typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ /* some universal 64 bit define... */ typedef long long int64; @@ -258,6 +260,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ + RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ + RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/vm.c b/runtime/vm.c index a25476c2..41d3e483 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -26,6 +26,7 @@ #include "config.h" #include #include +#include #include "rsyslog.h" #include "obj.h" @@ -39,6 +40,142 @@ DEFobjCurrIf(vmstk) DEFobjCurrIf(var) DEFobjCurrIf(sysvar) +/* ------------------------------ function registry code and structures ------------------------------ */ + +/* we maintain a registry of known functions */ +/* currently, this is a singly-linked list, this shall become a binary + * tree when we add the real call interface. So far, entries are added + * at the root, only. + */ +typedef struct s_rsf_entry { + cstr_t *pName; /* function name */ + prsf_t rsf; /* pointer to function code */ + struct s_rsf_entry *pNext; /* Pointer to next element or NULL */ +} rsf_entry_t; +rsf_entry_t *funcRegRoot = NULL; + + +/* add a function to the function registry. + * The handed-over cstr_t* object must no longer be used by the caller. + * A duplicate function name is an error. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsfrAddFunction(uchar *szName, prsf_t rsf) +{ + rsf_entry_t *pEntry; + size_t lenName; + DEFiRet; + + assert(szName != NULL); + assert(rsf != NULL); + + /* first check if we have a duplicate name, with the current approach this means + * we need to go through the whole list. + */ + lenName = strlen((char*)szName); + for(pEntry = funcRegRoot ; pEntry != NULL ; pEntry = pEntry->pNext) + if(!rsCStrSzStrCmp(pEntry->pName, szName, lenName)) + ABORT_FINALIZE(RS_RET_DUP_FUNC_NAME); + + /* unique name, so add to head of list */ + CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); + CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + pEntry->rsf = rsf; + pEntry->pNext = funcRegRoot; + funcRegRoot = pEntry; + +finalize_it: + RETiRet; +} + + +/* find a function inside the function registry + * The caller provides a cstr_t with the function name and receives + * a function pointer back. If no function is found, an RS_RET_UNKNW_FUNC + * error is returned. So if the function returns with RS_RET_OK, the caller + * can savely assume the function pointer is valid. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunction(cstr_t *pcsName, prsf_t *prsf) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(prsf != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(!rsCStrCStrCmp(pEntry->pName, pcsName)) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *prsf = pFound->rsf; + +finalize_it: + RETiRet; +} + + +/* find the name of a RainerScript function whom's function pointer + * is known. This function returns the cstr_t object, which MUST NOT + * be modified by the caller. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunctionName(prsf_t rsf, cstr_t **ppcsName) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(rsf != NULL); + assert(ppcsName != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(pEntry->rsf == rsf) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *ppcsName = pFound->pName; + +finalize_it: + RETiRet; +} + + +/* free the whole function registry + */ +static void +rsfrRemoveAll(void) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pEntryDel; + + BEGINfunc + pEntry = funcRegRoot; + while(pEntry != NULL) { + pEntryDel = pEntry; + pEntry = pEntry->pNext; + rsCStrDestruct(&pEntryDel->pName); + free(pEntryDel); + } + funcRegRoot = NULL; + ENDfunc +} + + +/* ------------------------------ end function registry code and structures ------------------------------ */ + /* ------------------------------ instruction set implementation ------------------------------ * * The following functions implement the VM's instruction set. @@ -330,7 +467,6 @@ CODESTARTop(PUSHSYSVAR) finalize_it: ENDop(PUSHSYSVAR) - /* The function call operation is only very roughly implemented. While the plumbing * to reach this instruction is fine, the instruction itself currently supports only * functions with a single argument AND with a name that we know. @@ -340,20 +476,9 @@ ENDop(PUSHSYSVAR) */ BEGINop(FUNC_CALL) /* remember to set the instruction also in the ENDop macro! */ var_t *numOperands; - var_t *operand1; - int iStrlen; CODESTARTop(FUNC_CALL) vmstk.PopNumber(pThis->pStk, &numOperands); - if(numOperands->val.num != 1) - ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); - vmstk.PopString(pThis->pStk, &operand1); /* guess there's just one ;) */ - if(!rsCStrSzStrCmp(pOp->operand.pVar->val.pStr, (uchar*) "strlen", 6)) { /* only one supported so far ;) */ -RUNLOG_VAR("%s", rsCStrGetSzStr(operand1->val.pStr)); - iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); -RUNLOG_VAR("%d", iStrlen); - } else - ABORT_FINALIZE(RS_RET_INVLD_FUNC); - PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME + CHKiRet((*pOp->operand.rsf)(pThis->pStk, numOperands->val.num)); var.Destruct(&numOperands); /* no longer needed */ finalize_it: ENDop(FUNC_CALL) @@ -362,6 +487,89 @@ ENDop(FUNC_CALL) /* ------------------------------ end instruction set implementation ------------------------------ */ +/* ------------------------------ begin built-in function implementation ------------------------------ */ +/* note: this shall probably be moved to a separate module, but for the time being we do it directly + * in here. This is on our way to get from a dirty to a clean solution via baby steps that are + * a bit less dirty each time... + * + * The advantage of doing it here is that we do not yet need to think about how to handle the + * exit case, where we must not unload function modules which functions are still referenced. + * + * CALLING INTERFACE: + * The function must pop its parameters off the stack and pop its result onto + * the stack when it is finished. The number of parameters the function was + * called with is provided to it. If the argument count is less then what the function + * expected, it may handle the situation with defaults (or return an error). If the + * argument count is greater than expected, returnung an error is highly + * recommended (use RS_RET_INVLD_NBR_ARGUMENTS for these cases). + * + * All function names are prefixed with "rsf_" (RainerScript Function) to have + * a separate "name space". + * + * rgerhards, 2009-04-06 + */ + + +/* The strlen function, also probably a prototype of how all functions should be + * implemented. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_strlen(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton (trivial case here...) */ + vmstk.PopString(pStk, &operand1); + iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); + + /* Store result and cleanup */ + var.SetNumber(operand1, iStrlen); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + +/* The "tolower" function, which converts its sole argument to lower case. + * Quite honestly, currently this is primarily a test driver for me... + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_tolower(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + uchar *pSrc; + cstr_t *pcstr; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton */ + CHKiRet(rsCStrConstruct(&pcstr)); + vmstk.PopString(pStk, &operand1); + pSrc = rsCStrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); + while(iStrlen--) { + CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + } + + /* Store result and cleanup */ + CHKiRet(rsCStrFinish(pcstr)); + var.SetString(operand1, pcstr); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(vm) /* be sure to specify the object type also in END macro! */ @@ -531,10 +739,23 @@ CODESTARTobjQueryInterface(vm) pIf->PopBoolFromStack = PopBoolFromStack; pIf->PopVarFromStack = PopVarFromStack; pIf->SetMsg = SetMsg; + pIf->FindRSFunction = findRSFunction; + pIf->FindRSFunctionName = findRSFunctionName; finalize_it: ENDobjQueryInterface(vm) +/* Exit the vm class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(vm, OBJ_IS_CORE_MODULE) /* class, version */ + rsfrRemoveAll(); + objRelease(sysvar, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vmstk, CORE_COMPONENT); +ENDObjClassExit(vm) + + /* Initialize the vm class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-02-19 @@ -548,6 +769,11 @@ BEGINObjClassInit(vm, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, vmDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmConstructFinalize); + + /* register built-in functions // TODO: move to its own module */ + CHKiRet(rsfrAddFunction((uchar*)"strlen", rsf_strlen)); + CHKiRet(rsfrAddFunction((uchar*)"tolower", rsf_tolower)); + ENDObjClassInit(vm) /* vi:set ai: diff --git a/runtime/vm.h b/runtime/vm.h index d2458220..cb3c69d0 100644 --- a/runtime/vm.h +++ b/runtime/vm.h @@ -55,8 +55,11 @@ BEGINinterface(vm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*PopBoolFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*PopVarFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*SetMsg)(vm_t *pThis, msg_t *pMsg); /* there are a few cases where we need this... */ + /* v2 (4.1.7) */ + rsRetVal (*FindRSFunction)(cstr_t *pcsName, prsf_t *prsf); /* 2009-06-04 */ + rsRetVal (*FindRSFunctionName)(prsf_t rsf, cstr_t **ppcsName); /* 2009-06-04 */ ENDinterface(vm) -#define vmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmop.c b/runtime/vmop.c index a343481e..3e001d27 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -32,10 +32,12 @@ #include "rsyslog.h" #include "obj.h" #include "vmop.h" +#include "vm.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(var) +DEFobjCurrIf(vm) /* forward definitions */ @@ -61,8 +63,10 @@ rsRetVal vmopConstructFinalize(vmop_t __attribute__((unused)) *pThis) /* destructor for the vmop object */ BEGINobjDestruct(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(vmop) - if(pThis->operand.pVar != NULL) - var.Destruct(&pThis->operand.pVar); + if(pThis->opcode != opcode_FUNC_CALL) { + if(pThis->operand.pVar != NULL) + var.Destruct(&pThis->operand.pVar); + } ENDobjDestruct(vmop) @@ -72,13 +76,19 @@ BEGINobjDebugPrint(vmop) /* be sure to specify the object type also in END and C cstr_t *pStrVar; CODESTARTobjDebugPrint(vmop) vmopOpcode2Str(pThis, &pOpcodeName); - CHKiRet(rsCStrConstruct(&pStrVar)); - CHKiRet(rsCStrFinish(&pStrVar)); - if(pThis->operand.pVar != NULL) { - CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pStrVar)); + assert(pStrVar != NULL); + } else { + CHKiRet(rsCStrConstruct(&pStrVar)); + if(pThis->operand.pVar != NULL) { + CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + } } + CHKiRet(rsCStrFinish(&pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); - rsCStrDestruct(&pStrVar); + if(pThis->opcode != opcode_FUNC_CALL) + rsCStrDestruct(&pStrVar); finalize_it: ENDobjDebugPrint(vmop) @@ -98,6 +108,7 @@ static rsRetVal Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) { uchar *pOpcodeName; + cstr_t *pcsFuncName; uchar szBuf[2048]; size_t lenBuf; DEFiRet; @@ -107,8 +118,13 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) vmopOpcode2Str(pThis, &pOpcodeName); lenBuf = snprintf((char*) szBuf, sizeof(szBuf), "%s\t", pOpcodeName); CHKiRet(rsCStrAppendStrWithLen(pstrPrg, szBuf, lenBuf)); - if(pThis->operand.pVar != NULL) - CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pcsFuncName)); + CHKiRet(rsCStrAppendCStr(pstrPrg, pcsFuncName)); + } else { + if(pThis->operand.pVar != NULL) + CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + } CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); finalize_it: @@ -116,6 +132,23 @@ finalize_it: } +/* set function + * rgerhards, 2009-04-06 + */ +static rsRetVal +vmopSetFunc(vmop_t *pThis, cstr_t *pcsFuncName) +{ + prsf_t rsf; /* pointer to function */ + DEFiRet; + ISOBJ_TYPE_assert(pThis, vmop); + CHKiRet(vm.FindRSFunction(pcsFuncName, &rsf)); /* check if function exists and obtain pointer to it */ + assert(rsf != NULL); /* just double-check, would be very hard to find! */ + pThis->operand.rsf = rsf; +finalize_it: + RETiRet; +} + + /* set operand (variant case) * rgerhards, 2008-02-20 */ @@ -248,6 +281,7 @@ CODESTARTobjQueryInterface(vmop) pIf->ConstructFinalize = vmopConstructFinalize; pIf->Destruct = vmopDestruct; pIf->DebugPrint = vmopDebugPrint; + pIf->SetFunc = vmopSetFunc; pIf->SetOpcode = vmopSetOpcode; pIf->SetVar = vmopSetVar; pIf->Opcode2Str = vmopOpcode2Str; @@ -263,6 +297,7 @@ ENDobjQueryInterface(vmop) BEGINObjClassInit(vmop, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); OBJSetMethodHandler(objMethod_DEBUGPRINT, vmopDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmopConstructFinalize); diff --git a/runtime/vmop.h b/runtime/vmop.h index 938b08fd..67048c26 100644 --- a/runtime/vmop.h +++ b/runtime/vmop.h @@ -26,6 +26,7 @@ #define INCLUDED_VMOP_H #include "ctok_token.h" +#include "vmstk.h" #include "stringbuf.h" /* machine instructions types */ @@ -96,7 +97,8 @@ typedef struct vmop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ opcode_t opcode; union { - var_t *pVar; /* for function call, this is the name (string) of function to be called */ + var_t *pVar; + prsf_t rsf; /* pointer to function for "call" instruction */ } operand; struct vmop_s *pNext; /* next operation or NULL, if end of program (logically this belongs to vmprg) */ } vmop_t; @@ -112,8 +114,13 @@ BEGINinterface(vmop) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetVar)(vmop_t *pThis, var_t *pVar); rsRetVal (*Opcode2Str)(vmop_t *pThis, uchar **ppName); rsRetVal (*Obj2Str)(vmop_t *pThis, cstr_t *pstr); + /* v2 */ + rsRetVal (*SetFunc)(vmop_t *pThis, cstr_t *pcsFuncName); ENDinterface(vmop) -#define vmopCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmopCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* interface changes, v1 -> v2 + * added SetFuct after existing function pointers -- rgerhards, 2009-04-06 + */ /* the remaining prototypes */ PROTOTYPEObj(vmop); diff --git a/runtime/vmprg.c b/runtime/vmprg.c index 75915025..07757b98 100644 --- a/runtime/vmprg.c +++ b/runtime/vmprg.c @@ -155,7 +155,6 @@ vmprgAddVarOperation(vmprg_t *pThis, opcode_t opcode, var_t *pVar) /* construct and fill vmop */ CHKiRet(vmop.Construct(&pOp)); CHKiRet(vmop.ConstructFinalize(pOp)); - CHKiRet(vmop.ConstructFinalize(pOp)); CHKiRet(vmop.SetOpcode(pOp, opcode)); if(pVar != NULL) CHKiRet(vmop.SetVar(pOp, pVar)); @@ -168,6 +167,32 @@ finalize_it: } +/* this is another shortcut for high-level callers. It is similar to vmprgAddVarOperation + * but adds a call operation. Among others, this include a check if the function + * is known. + */ +static rsRetVal +vmprgAddCallOperation(vmprg_t *pThis, cstr_t *pcsName) +{ + DEFiRet; + vmop_t *pOp; + + ISOBJ_TYPE_assert(pThis, vmprg); + + /* construct and fill vmop */ + CHKiRet(vmop.Construct(&pOp)); + CHKiRet(vmop.ConstructFinalize(pOp)); + CHKiRet(vmop.SetFunc(pOp, pcsName)); + CHKiRet(vmop.SetOpcode(pOp, opcode_FUNC_CALL)); + + /* and add it to the program */ + CHKiRet(vmprgAddOperation(pThis, pOp)); + +finalize_it: + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-02-21 */ @@ -189,6 +214,7 @@ CODESTARTobjQueryInterface(vmprg) pIf->Obj2Str = Obj2Str; pIf->AddOperation = vmprgAddOperation; pIf->AddVarOperation = vmprgAddVarOperation; + pIf->AddCallOperation = vmprgAddCallOperation; finalize_it: ENDobjQueryInterface(vmprg) diff --git a/runtime/vmprg.h b/runtime/vmprg.h index c1042f7d..66f03913 100644 --- a/runtime/vmprg.h +++ b/runtime/vmprg.h @@ -57,8 +57,10 @@ BEGINinterface(vmprg) /* name must also be changed in ENDinterface macro! */ rsRetVal (*AddOperation)(vmprg_t *pThis, vmop_t *pOp); rsRetVal (*AddVarOperation)(vmprg_t *pThis, opcode_t opcode, var_t *pVar); rsRetVal (*Obj2Str)(vmprg_t *pThis, cstr_t *pstr); + /* v2 (4.1.7) */ + rsRetVal (*AddCallOperation)(vmprg_t *pThis, cstr_t *pVar); /* added 2009-04-06 */ ENDinterface(vmprg) -#define vmprgCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmprgCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmstk.h b/runtime/vmstk.h index 2d45ee4d..06657cf4 100644 --- a/runtime/vmstk.h +++ b/runtime/vmstk.h @@ -27,11 +27,11 @@ #define VMSTK_SIZE 256 /* the vmstk object */ -typedef struct vmstk_s { +struct vmstk_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ var_t *vStk[VMSTK_SIZE];/* the actual stack */ int iStkPtr; /* stack pointer, points to next free location, grows from 0 --> topend */ -} vmstk_t; +}; /* interfaces */ diff --git a/tests/3.rstest b/tests/3.rstest index 93cb941a..e75d9754 100644 --- a/tests/3.rstest +++ b/tests/3.rstest @@ -7,10 +7,10 @@ out: 00000000: push_msgvar msg[cstr] 00000001: push_const abc[cstr] 00000002: push_const 1[nbr] -00000003: func_call strlen[cstr] +00000003: func_call strlen 00000004: strconcat 00000005: push_const 1[nbr] -00000006: func_call strlen[cstr] +00000006: func_call strlen 00000007: push_const 20[nbr] 00000008: push_const 30[nbr] 00000009: add diff --git a/tools/syslogd.c b/tools/syslogd.c index b23c12a7..c85c4371 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3091,6 +3091,7 @@ GlobalClassExit(void) objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ objRelease(conf, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); + vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ objRelease(vm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); -- cgit From 7db9f96fe9ecb9f8e05d45cc888aa488d8aed85f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 18:07:52 +0200 Subject: testcase added (on user request) --- tests/testsuites/3.parse1 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/testsuites/3.parse1 diff --git a/tests/testsuites/3.parse1 b/tests/testsuites/3.parse1 new file mode 100644 index 00000000..a6b4e884 --- /dev/null +++ b/tests/testsuites/3.parse1 @@ -0,0 +1,3 @@ +<38>Apr 6 15:07:10 lxcvs07 sshd(pam_unix)[31738]: session closed for user cvsadmin +38,auth,info,Apr 6 15:07:10,lxcvs07,sshd(pam_unix),sshd(pam_unix)[31738]:, session closed for user cvsadmin +# yet another real-life sample where we had some issues with -- cgit From 8f8e2cd66bd7f1e35f7bf0678e25bbdb99d67093 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 12:19:54 +0200 Subject: improved testbench, added tests for tcp-based reception --- ChangeLog | 1 + tests/Makefile.am | 6 +- tests/nettester.c | 392 ++++++++++++++++++++++++++++++++++++ tests/omod-if-array.sh | 6 +- tests/parsertest.sh | 6 +- tests/testsuites/omod-if-array.conf | 3 +- tests/testsuites/parse1.conf | 3 +- tests/udptester.c | 293 --------------------------- 8 files changed, 408 insertions(+), 302 deletions(-) create mode 100644 tests/nettester.c delete mode 100644 tests/udptester.c diff --git a/ChangeLog b/ChangeLog index 821d9439..98146600 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" +- improved testbench, added tests for tcp-based reception --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/tests/Makefile.am b/tests/Makefile.am index ab1c5a62..fa662a3e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,5 +1,5 @@ TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail udptester +check_PROGRAMS = $(TESTRUNS) ourtail nettester TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid @@ -32,8 +32,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ ourtail_SOURCES = ourtail.c -udptester_SOURCES = udptester.c getline.c -udptester_LDADD = $(SOL_LIBS) +nettester_SOURCES = nettester.c getline.c +nettester_LDADD = $(SOL_LIBS) rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/nettester.c b/tests/nettester.c new file mode 100644 index 00000000..89a784f3 --- /dev/null +++ b/tests/nettester.c @@ -0,0 +1,392 @@ +/* Runs a test suite on the rsyslog (and later potentially + * other things). + * + * The name of the test suite must be given as argv[1]. In this config, + * rsyslogd is loaded with config ./testsuites/.conf and then + * test cases ./testsuites/ *. are executed on it. This test driver is + * suitable for testing cases where a message sent (via UDP) results in + * exactly one response. It can not be used in cases where no response + * is expected (that would result in a hang of the test driver). + * Note: each test suite can contain many tests, but they all need to work + * with the same rsyslog configuration. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static enum { inputUDP, inputTCP } inputMode; /* input for which tests are to be run */ +static pid_t rsyslogdPid = 0; /* pid of rsyslog instance being tested */ +static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ +static char *testSuite; /* name of current test suite */ + + +void readLine(int fd, char *ln) +{ + char c; + int lenRead; + lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + *ln++ = c; + lenRead = read(fd, &c, 1); + } + *ln = '\0'; +} + + +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + +/* send a message via UDP + * returns 0 if ok, something else otherwise. + */ +int +udpSend(char *buf, int lenBuf) +{ + struct sockaddr_in si_other; + int s, slen=sizeof(si_other); + + if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &si_other, 0, sizeof(si_other)); + si_other.sin_family = AF_INET; + si_other.sin_port = htons(12514); + if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + + if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { + perror("sendto"); + fprintf(stderr, "sendto() failed\n"); + return(1); + } + + close(s); + return 0; +} + + +/* open pipe to test candidate - so far, this is + * always rsyslogd and with a fixed config. Later, we may + * change this. Returns 0 if ok, something else otherwise. + * rgerhards, 2009-03-31 + */ +int openPipe(char *configFile, pid_t *pid, int *pfd) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", + "-M../runtime/.libs:../.libs", NULL }; + char confFile[1024]; + char *newenviron[] = { NULL }; + + + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); + newargv[1] = confFile; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if(cpid == 0) { /* Child reads from pipe */ + fclose(stdout); + dup(pipefd[1]); + close(pipefd[1]); + close(pipefd[0]); + fclose(stdin); + execve("../tools/rsyslogd", newargv, newenviron); + } else { + close(pipefd[1]); + *pid = cpid; + *pfd = pipefd[0]; + } + + return(0); +} + + +/* Process a specific test case. File name is provided. + * Needs to return 0 if all is OK, something else otherwise. + */ +int +processTestFile(int fd, char *pszFileName) +{ + FILE *fp; + char *testdata = NULL; + char *expected = NULL; + int ret = 0; + size_t lenLn; + char buf[4096]; + + if((fp = fopen((char*)pszFileName, "r")) == NULL) { + perror((char*)pszFileName); + return(2); + } + + /* skip comments at start of file */ + + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + + + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send (we could use function pointers here...) */ + if(inputMode == inputUDP) { + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + } else { + if(tcpSend(testdata, strlen(testdata)) != 0) + return(2); + } + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + free(testdata); + free(expected); + fclose(fp); + return(ret); +} + + +/* carry out all tests. Tests are specified via a file name + * wildcard. Each of the files is read and the test carried + * out. + * Returns the number of tests that failed. Zero means all + * success. + */ +int +doTests(int fd, char *files) +{ + int iFailed = 0; + int iTests = 0; + int ret; + char *testFile; + glob_t testFiles; + size_t i = 0; + struct stat fileInfo; + + glob(files, GLOB_MARK, NULL, &testFiles); + + for(i = 0; i < testFiles.gl_pathc; i++) { + testFile = testFiles.gl_pathv[i]; + + if(stat((char*) testFile, &fileInfo) != 0) + continue; /* continue with the next file if we can't stat() the file */ + + ++iTests; + /* all regular files are run through the test logic. Symlinks don't work. */ + if(S_ISREG(fileInfo.st_mode)) { /* config file */ + printf("processing test case '%s' ... ", testFile); + ret = processTestFile(fd, testFile); + if(ret == 0) { + printf("successfully completed\n"); + } else { + printf("failed!\n"); + ++iFailed; + } + } + } + globfree(&testFiles); + + if(iTests == 0) { + printf("Error: no test cases found, no tests executed.\n"); + iFailed = 1; + } else { + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + } + + return(iFailed); +} + +/* cleanup */ +void doAtExit(void) +{ + int status; + + if(rsyslogdPid != 0) { + kill(rsyslogdPid, SIGTERM); + waitpid(rsyslogdPid, &status, 0); /* wait until instance terminates */ + } + + unlink(NETTEST_INPUT_CONF_FILE); +} + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int fd; + int ret = 0; + FILE *fp; + char buf[4096]; + char testcases[4096]; + + if(argc != 3) { + printf("Invalid call of nettester\n"); + printf("Usage: nettester testsuite-name input\n"); + printf(" input = udp|tcp\n"); + exit(1); + } + + atexit(doAtExit); + + testSuite = argv[1]; + + if(!strcmp(argv[2], "udp")) + inputMode = inputUDP; + else if(!strcmp(argv[2], "tcp")) + inputMode = inputTCP; + else { + printf("error: unsupported input mode '%s'\n", argv[2]); + exit(1); + } + + if((srcdir = getenv("srcdir")) == NULL) + srcdir = "."; + + printf("Start of nettester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + + /* create input config file */ + if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { + perror(NETTEST_INPUT_CONF_FILE); + printf("error opening input configuration file\n"); + exit(1); + } + if(inputMode == inputUDP) { + fputs("$ModLoad ../plugins/imudp/.libs/imudp\n", fp); + fputs("$UDPServerRun 12514\n", fp); + } else { + fputs("$ModLoad ../plugins/imtcp/.libs/imtcp\n", fp); + fputs("$InputTCPServerRun 13514\n", fp); + } + fclose(fp); + + /* start to be tested rsyslogd */ + openPipe(argv[1], &rsyslogdPid, &fd); + readLine(fd, buf); + + /* generate filename */ + sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); + if(doTests(fd, testcases) != 0) + ret = 1; + + printf("End of nettester run (%d).\n", ret); + exit(ret); +} diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index cac08928..8a8d67f3 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1 +1,5 @@ -./udptester omod-if-array +#!/bin/bash -e +echo test omod-if-array via udp +./nettester omod-if-array udp +echo test omod-if-array via tcp +./nettester omod-if-array tcp diff --git a/tests/parsertest.sh b/tests/parsertest.sh index e7985bb0..fabe7e8d 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1 +1,5 @@ -./udptester parse1 +#!/bin/bash -e +echo test parsertest via udp +./nettester parse1 udp +echo test parsertest via tcp +./nettester parse1 tcp diff --git a/tests/testsuites/omod-if-array.conf b/tests/testsuites/omod-if-array.conf index e6c05a52..d88db166 100644 --- a/tests/testsuites/omod-if-array.conf +++ b/tests/testsuites/omod-if-array.conf @@ -3,8 +3,7 @@ # the testbench, so we do not need to focus on that) # rgerhards, 2009-04-03 $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ActionOMStdoutArrayInterface on $ErrorMessagesToStderr off diff --git a/tests/testsuites/parse1.conf b/tests/testsuites/parse1.conf index 0fb7d16d..947a05a8 100644 --- a/tests/testsuites/parse1.conf +++ b/tests/testsuites/parse1.conf @@ -1,6 +1,5 @@ $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ErrorMessagesToStderr off diff --git a/tests/udptester.c b/tests/udptester.c deleted file mode 100644 index a3c6658d..00000000 --- a/tests/udptester.c +++ /dev/null @@ -1,293 +0,0 @@ -/* Runs a test suite on the rsyslog (and later potentially - * other things). - * - * The name of the test suite must be given as argv[1]. In this config, - * rsyslogd is loaded with config ./testsuites/.conf and then - * test cases ./testsuites/ *. are executed on it. This test driver is - * suitable for testing cases where a message sent (via UDP) results in - * exactly one response. It can not be used in cases where no response - * is expected (that would result in a hang of the test driver). - * Note: each test suite can contain many tests, but they all need to work - * with the same rsyslog configuration. - * - * Part of the testbench for rsyslog. - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define EXIT_FAILURE 1 - -static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ -static char *testSuite; /* name of current test suite */ - - -void readLine(int fd, char *ln) -{ - char c; - int lenRead; - lenRead = read(fd, &c, 1); - while(lenRead == 1 && c != '\n') { - *ln++ = c; - lenRead = read(fd, &c, 1); - } - *ln = '\0'; -} - - -/* send a message via UDP - * returns 0 if ok, something else otherwise. - */ -int -udpSend(char *buf, int lenBuf) -{ - struct sockaddr_in si_other; - int s, slen=sizeof(si_other); - - if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { - perror("socket()"); - return(1); - } - - memset((char *) &si_other, 0, sizeof(si_other)); - si_other.sin_family = AF_INET; - si_other.sin_port = htons(12514); - if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { - fprintf(stderr, "inet_aton() failed\n"); - return(1); - } - - if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { - perror("sendto"); - fprintf(stderr, "sendto() failed\n"); - return(1); - } - - close(s); - return 0; -} - - -/* open pipe to test candidate - so far, this is - * always rsyslogd and with a fixed config. Later, we may - * change this. Returns 0 if ok, something else otherwise. - * rgerhards, 2009-03-31 - */ -int openPipe(char *configFile, pid_t *pid, int *pfd) -{ - int pipefd[2]; - pid_t cpid; - char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime//.libs", NULL }; - char confFile[1024]; - char *newenviron[] = { NULL }; - - - sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); - newargv[1] = confFile; - - if (pipe(pipefd) == -1) { - perror("pipe"); - exit(EXIT_FAILURE); - } - - cpid = fork(); - if (cpid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } - - if(cpid == 0) { /* Child reads from pipe */ - fclose(stdout); - dup(pipefd[1]); - close(pipefd[1]); - close(pipefd[0]); - fclose(stdin); - execve("../tools/rsyslogd", newargv, newenviron); - } else { - close(pipefd[1]); - *pid = cpid; - *pfd = pipefd[0]; - } - - return(0); -} - - -/* Process a specific test case. File name is provided. - * Needs to return 0 if all is OK, something else otherwise. - */ -int -processTestFile(int fd, char *pszFileName) -{ - FILE *fp; - char *testdata = NULL; - char *expected = NULL; - int ret = 0; - size_t lenLn; - char buf[4096]; - - if((fp = fopen((char*)pszFileName, "r")) == NULL) { - perror((char*)pszFileName); - return(2); - } - - /* skip comments at start of file */ - - getline(&testdata, &lenLn, fp); - while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } - - - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send */ - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ - - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; - } - - free(testdata); - free(expected); - fclose(fp); - return(ret); -} - - -/* carry out all tests. Tests are specified via a file name - * wildcard. Each of the files is read and the test carried - * out. - * Returns the number of tests that failed. Zero means all - * success. - */ -int -doTests(int fd, char *files) -{ - int iFailed = 0; - int iTests = 0; - int ret; - char *testFile; - glob_t testFiles; - size_t i = 0; - struct stat fileInfo; - - glob(files, GLOB_MARK, NULL, &testFiles); - - for(i = 0; i < testFiles.gl_pathc; i++) { - testFile = testFiles.gl_pathv[i]; - - if(stat((char*) testFile, &fileInfo) != 0) - continue; /* continue with the next file if we can't stat() the file */ - - ++iTests; - /* all regular files are run through the test logic. Symlinks don't work. */ - if(S_ISREG(fileInfo.st_mode)) { /* config file */ - printf("processing test case '%s' ... ", testFile); - ret = processTestFile(fd, testFile); - if(ret == 0) { - printf("successfully completed\n"); - } else { - printf("failed!\n"); - ++iFailed; - } - } - } - globfree(&testFiles); - - if(iTests == 0) { - printf("Error: no test cases found, no tests executed.\n"); - iFailed = 1; - } else { - printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); - } - - return(iFailed); -} - - -/* Run the test suite. This must be called with exactly one parameter, the - * name of the test suite. For details, see file header comment at the top - * of this file. - * rgerhards, 2009-04-03 - */ -int main(int argc, char *argv[]) -{ - int fd; - pid_t pid; - int status; - int ret = 0; - char buf[4096]; - char testcases[4096]; - - if(argc != 2) { - printf("Invalid call of udptester\n"); - printf("Usage: udptester testsuite-name\n"); - exit(1); - } - - testSuite = argv[1]; - - if((srcdir = getenv("srcdir")) == NULL) - srcdir = "."; - - printf("Start of udptester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); - - openPipe(argv[1], &pid, &fd); - readLine(fd, buf); - - /* generate filename */ - sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); - if(doTests(fd, testcases) != 0) - ret = 1; - - /* cleanup */ - kill(pid, SIGTERM); - waitpid(pid, &status, 0); /* wait until instance terminates */ - printf("End of udptester run.\n"); - exit(ret); -} -- cgit From 7cc7166cb24e50e48cab1cbb3c19c8c17d95ace7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 17:48:02 +0200 Subject: added new test case for many tcp connections It is checked that many tcp connections are properly handled. While adding this test, I noticed that there is a bug in imtcp that prevents creation of more than 200 connections. This bug still exists, so the test suite currently fails (what is correct). Will be addressed soon. --- ChangeLog | 4 +- tests/Makefile.am | 7 +- tests/chkseq.c | 76 +++++++++++++++++ tests/manytcp.sh | 13 +++ tests/tcpflood | Bin 0 -> 17940 bytes tests/tcpflood.c | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 tests/chkseq.c create mode 100755 tests/manytcp.sh create mode 100755 tests/tcpflood create mode 100644 tests/tcpflood.c diff --git a/ChangeLog b/ChangeLog index 98146600..98b4f235 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,7 +5,9 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" -- improved testbench, added tests for tcp-based reception +- improved testbench + * added tests for tcp-based reception + * added tcp-load test (1000 connections, 20,000 messages) --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/tests/Makefile.am b/tests/Makefile.am index fa662a3e..53a81a93 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -27,10 +27,13 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ parsertest.sh \ + manytcp.sh \ omod-if-array.sh \ cfg.sh ourtail_SOURCES = ourtail.c +tcpflood_SOURCES = tcpflood.c +chkseq_SOURCES = chkseq.c nettester_SOURCES = nettester.c getline.c nettester_LDADD = $(SOL_LIBS) diff --git a/tests/chkseq.c b/tests/chkseq.c new file mode 100644 index 00000000..3203c250 --- /dev/null +++ b/tests/chkseq.c @@ -0,0 +1,76 @@ +/* Checks if a file consists of line of strictly monotonically + * increasing numbers. An expected start and end number may + * be set. + * + * Params + * argv[1] file to check + * argv[2] start number + * argv[3] end number + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include + +int main(int argc, char *argv[]) +{ + FILE *fp; + int val; + int i; + int ret = 0; + int start, end; + + if(argc != 4) { + printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file start end\n"); + exit(1); + } + + start = atoi(argv[2]); + end = atoi(argv[3]); + + if(start > end) { + printf("start must be less than or equal end!\n"); + exit(1); + } + + /* read file */ + fp = fopen(argv[1], "r"); + if(fp == NULL) { + perror(argv[1]); + exit(1); + } + + for(i = start ; i < end ; ++i) { + if(fscanf(fp, "%d\n", &val) != 1) { + printf("scanf error in index i=%d\n", i); + exit(1); + } + if(val != i) { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } + } + + exit(ret); +} diff --git a/tests/manytcp.sh b/tests/manytcp.sh new file mode 100755 index 00000000..3accfb8a --- /dev/null +++ b/tests/manytcp.sh @@ -0,0 +1,13 @@ +rm -f rsyslog.out.log # work file +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -ftestsuites/manytcp.conf & +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13514 1000 20000 +sleep 1 +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 19999 +if [ "$?" -ne "0" ]; then + echo "sequence error detected" + exit 1 +fi diff --git a/tests/tcpflood b/tests/tcpflood new file mode 100755 index 00000000..90b89b84 Binary files /dev/null and b/tests/tcpflood differ diff --git a/tests/tcpflood.c b/tests/tcpflood.c new file mode 100644 index 00000000..cecec628 --- /dev/null +++ b/tests/tcpflood.c @@ -0,0 +1,245 @@ +/* Opens a large number of tcp connections and sends + * messages over them. This is used for stress-testing. + * + * Params + * argv[1] target address + * argv[2] target port + * argv[3] number of connections + * argv[4] number of messages to send (connection is random) + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static char *targetIP; +static int targetPort; +static int numMsgsToSend; /* number of messages to send */ +static int numConnections; /* number of connections to create */ +static int *sockArray; /* array of sockets to use */ + + +/* open a single tcp connection + */ +int openConn(int *fd) +{ + int sock; + struct sockaddr_in addr; + + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(targetPort); + if(inet_aton(targetIP, &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + perror("connect()"); + fprintf(stderr, "connect() failed\n"); + return(1); + } + + *fd = sock; + return 0; +} + + +/* open all requested tcp connections + * this includes allocating the connection array + */ +int openConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " open connections", sizeof(" open connections")-1); + sockArray = calloc(numConnections, sizeof(int)); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); + write(1, msgBuf, lenMsg); + } + if(openConn(&(sockArray[i])) != 0) { + printf("error in trying to open connection i=%d\n", i); + return 1; + } + } + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d open connections\n", i); + write(1, msgBuf, lenMsg); + + return 0; +} + + +/* send messages to the tcp connections we keep open. We use + * a very basic format that helps identify the message + * (via msgnum:: e.g. msgnum:00000001:). This format is suitable + * for extracton to field-based properties. + * The first numConnection messages are sent sequentially, as are the + * last. All messages in between are sent over random connections. + * Note that message numbers start at 0. + */ +int sendMessages(void) +{ + int i; + int socknum; + int lenBuf; + char buf[2048]; + char msgBuf[128]; + size_t lenMsg; + + srand(time(NULL)); /* seed is good enough for our needs */ + + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent", 0); + write(1, msgBuf, lenMsg); + for(i = 0 ; i < numMsgsToSend ; ++i) { + if(i < numConnections) + socknum = i; + else if(i >= numMsgsToSend - numConnections) + socknum = i - (numMsgsToSend - numConnections); + else + socknum = rand() % numConnections; + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + if(send(sockArray[socknum], buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + if(i % 100 == 0) { + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); + write(1, msgBuf, lenMsg); + } + } + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent\n", i); + write(1, msgBuf, lenMsg); + + return 0; +} + + +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int ret = 0; + static char buf[1024]; + + setvbuf(stdout, _IONBF, buf, 48); + + if(argc != 5) { + printf("Invalid call of tcpflood\n"); + printf("Usage: nettester testsuite-name input\n"); + exit(1); + } + + targetIP = argv[1]; + targetPort = atoi(argv[2]); + numConnections = atoi(argv[3]); + numMsgsToSend = atoi(argv[4]); + + if(openConnections() != 0) { + printf("error opening connections\n"); + exit(1); + } + + if(sendMessages() != 0) { + printf("error sending messages\n"); + exit(1); + } + exit(ret); +} -- cgit From de38f744de1ce746e6e61098fd4a05ac76ef71a0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 18:18:31 +0200 Subject: minor cleanup --- tests/tcpflood | Bin 17940 -> 17972 bytes tests/tcpflood.c | 2 +- tools/syslogd.c | 8 -------- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/tcpflood b/tests/tcpflood index 90b89b84..ae00fcd5 100755 Binary files a/tests/tcpflood and b/tests/tcpflood differ diff --git a/tests/tcpflood.c b/tests/tcpflood.c index cecec628..83f0d1ee 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -223,7 +223,7 @@ int main(int argc, char *argv[]) if(argc != 5) { printf("Invalid call of tcpflood\n"); - printf("Usage: nettester testsuite-name input\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); exit(1); } diff --git a/tools/syslogd.c b/tools/syslogd.c index 0badac19..a4f0059b 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3591,18 +3591,10 @@ int realMain(int argc, char **argv) fprintf(stderr, "-t option only supported in compatibility modes 0 to 2 - ignored\n"); break; case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ -{ -char buf[1024]; -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -printf("chroot to '%s'\n", arg); if(chroot(arg) != 0) { perror("chroot"); exit(1); } -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -} break; case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); -- cgit From 1cfa08749b1c474de850f693915b9a32d456c593 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 18:59:15 +0200 Subject: bugfix: solved potential memory leak in msg processing could manifest itself in imtcp (loss of a few bytes for *each* received message - but depended on config) -- this was newly introduced --- ChangeLog | 2 ++ runtime/msg.c | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 98146600..c99cc00d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,8 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? of loadable library modules. - added new RainerScript function "tolower" - improved testbench, added tests for tcp-based reception +- bugfix: solved potential memory leak in msg processing, could manifest + itself in imtcp --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/runtime/msg.c b/runtime/msg.c index 5d1f21fd..9d5f3838 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1542,6 +1542,8 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) { assert(pMsg != NULL); assert(pBuf != NULL); + if(pMsg->pszHOSTNAME != NULL) + free(pMsg->pszHOSTNAME); pMsg->iLenHOSTNAME = strlen(pBuf); pMsg->pszHOSTNAME = (uchar*) pBuf; } @@ -1567,7 +1569,7 @@ void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME) if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); else - dbgprintf("Could not allocate memory in MsgSetHOSTNAME()\n"); + DBGPRINTF("Could not allocate memory in MsgSetHOSTNAME()\n"); } -- cgit From d7f11ecb06688186d4c68b5933fb1437279ce03d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 8 Apr 2009 16:50:57 +0200 Subject: Make it work in batches of statements. Currently, all statements to be executed are stored on the same structure. When the batch size is reached, all statements are executed in a single transaction, and then committed. There are many corner cases in which an error may happen and the batch may be left in an inconsistent state, perhaps leaking memory or crashing. They will be fixed. --- plugins/omoracle/omoracle.c | 71 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ea910d3a..9b1ec42d 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -15,7 +15,8 @@ $OmoracleDBPassword: password to log in on the database. $OmoracleDB: connection string (an Oracle easy connect or a db name as specified by tnsnames.ora) - + $OmoracleBatchSize: Number of elements to send to the DB. + All fields are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -50,6 +51,18 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +/** */ +struct oracle_batch +{ + /* Batch size */ + int size; + /* Last element inserted in the buffer. The batch will be + * executed when n == size */ + int n; + /* Statements to run on this transaction */ + char** statements; +}; + typedef struct _instanceData { /* Environment handler, the base for any OCI work. */ OCIEnv* environment; @@ -67,6 +80,8 @@ typedef struct _instanceData { OCIBind* binding; /* Connection string, kept here for possible retries. */ char* connection; + /* Batch */ + struct oracle_batch batch; } instanceData; /** Database name, to be filled by the $OmoracleDB directive */ @@ -76,6 +91,8 @@ static char* db_name; static char* db_user; /** Database password, to be filled by the $OmoracleDBPassword */ static char* db_password; +/** Batch size. */ +static int batch_size; /** Generic function for handling errors from OCI. @@ -148,6 +165,12 @@ CODESTARTcreateInstance OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); + pData->batch.n = 0; + pData->batch.size = batch_size; + pData->batch.statements = calloc(pData->batch.size, + sizeof *pData->batch.statements); + CHKmalloc(pData->batch.statements); + finalize_it: ENDcreateInstance @@ -163,6 +186,9 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); + while (pData->batch.size--) + free(pData->batch.statements[pData->batch.size]); + free(pData->batch.statements); dbgprintf ("omoracle freed all its resources\n"); RETiRet; @@ -263,21 +289,41 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction + int i; + int n; CODESTARTdoAction dbgprintf("omoracle attempting to execute statement %s\n", *ppString); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, pData->error, *ppString, - strlen(*ppString), OCI_NTV_SYNTAX, - OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, pData->statement, pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); + + if (pData->batch.n == pData->batch.size) { + dbgprintf("omoracle batch size limit hit, sending into DB\n"); + for (i = 0; i < pData->batch.n; i++) { + if (pData->batch.statements[i] == NULL) + continue; + n = strlen(pData->batch.statements[i]); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->batch.statements[i], n, + OCI_NTV_SYNTAX, OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + free(pData->batch.statements[i]); + pData->batch.statements[i] = NULL; + } + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); + pData->batch.n = 0; + } + pData->batch.statements[pData->batch.n] = strdup(*ppString); + CHKmalloc(pData->batch.statements[pData->batch.n]); + pData->batch.n++; + finalize_it: dbgprintf ("omoracle %s at executing statement %s\n", iRet?"did not succeed":"succeeded", *ppString); -/* Clean credentials to avoid leakage in case of core dump. */ ENDdoAction BEGINmodExit @@ -331,4 +377,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, eCmdHdlrGetWord, NULL, &db_name, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchsize", 0, + eCmdHdlrInt, NULL, &batch_size, + STD_LOADABLE_MODULE_ID)); ENDmodInit -- cgit From 9bbd5dfd25f3e2a7a6839ff0e7b4076186efbcf4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 8 Apr 2009 16:50:58 +0200 Subject: Solve a memory leak when freeing Oracle instances. --- plugins/omoracle/omoracle.c | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 9b1ec42d..df06a8b6 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -190,7 +190,6 @@ CODESTARTfreeInstance free(pData->batch.statements[pData->batch.size]); free(pData->batch.statements); dbgprintf ("omoracle freed all its resources\n"); - RETiRet; ENDfreeInstance -- cgit From 1ebbd3a2df09ed9e706be7762307cd17c4a5cad0 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 8 Apr 2009 16:50:59 +0200 Subject: Stop omoracle losing messages on rsyslog shutdown. When rsyslog shuts down, we must send and commit any pending messages or information will be lost. It will make rsyslog's shut down slower, but also more reliable. --- plugins/omoracle/omoracle.c | 65 +++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index df06a8b6..f6679953 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -12,13 +12,17 @@ namely: $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db name as specified by tnsnames.ora) - $OmoracleBatchSize: Number of elements to send to the DB. - All fields are mandatory. The dbstring can be an Oracle easystring - or a DB name, as present in the tnsnames.ora file. + $OmoracleBatchSize: Number of elements to send to the DB on each + transaction. + + All these directives are mandatory. The dbstring can be an Oracle + easystring or a DB name, as present in the tnsnames.ora file. Author: Luis Fernando Muñoz Mejías @@ -174,11 +178,45 @@ CODESTARTcreateInstance finalize_it: ENDcreateInstance +/* Inserts all stored statements into the database, releasing any + * allocated memory. */ +static int insert_to_db(instanceData* pData) +{ + DEFiRet; + int i, n; + + for (i = 0; i < pData->batch.n; i++) { + if (pData->batch.statements[i] == NULL) + continue; + n = strlen(pData->batch.statements[i]); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->batch.statements[i], n, + OCI_NTV_SYNTAX, OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + free(pData->batch.statements[i]); + pData->batch.statements[i] = NULL; + } + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); + pData->batch.n = 0; +finalize_it: + RETiRet; +} + /** Close the session and free anything allocated by createInstance. */ BEGINfreeInstance CODESTARTfreeInstance +/* Before actually releasing our resources, let's try to commit + * anything pending so that we don't lose any messages. */ + insert_to_db(pData); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->environment, OCI_HTYPE_ENV); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); @@ -295,26 +333,7 @@ CODESTARTdoAction if (pData->batch.n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); - for (i = 0; i < pData->batch.n; i++) { - if (pData->batch.statements[i] == NULL) - continue; - n = strlen(pData->batch.statements[i]); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, - pData->error, - pData->batch.statements[i], n, - OCI_NTV_SYNTAX, OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, - pData->statement, - pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - free(pData->batch.statements[i]); - pData->batch.statements[i] = NULL; - } - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); - pData->batch.n = 0; + CHKiRet(insert_to_db(pData)); } pData->batch.statements[pData->batch.n] = strdup(*ppString); CHKmalloc(pData->batch.statements[pData->batch.n]); -- cgit From cf0cf2fa4bf84188baf873522d4a22887f4fdf90 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 13:57:35 +0200 Subject: removed accidently added binary --- tests/tcpflood | Bin 17972 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 tests/tcpflood diff --git a/tests/tcpflood b/tests/tcpflood deleted file mode 100755 index ae00fcd5..00000000 Binary files a/tests/tcpflood and /dev/null differ -- cgit From 1e52abd8be2dfde8727c7e54b02a10ce5051815f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 14:14:00 +0200 Subject: fixing "make distcheck" --- tests/testsuites/manytcp.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/testsuites/manytcp.conf diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf new file mode 100644 index 00000000..e491cd04 --- /dev/null +++ b/tests/testsuites/manytcp.conf @@ -0,0 +1,11 @@ +# Test for tcp "flood" testing +# rgerhards, 2009-04-08 +$ModLoad ../plugins/imtcp/.libs/imtcp +$inputtcpmaxsessions 2000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit From d2d8cc9c1b65bedeafaa3a06ed53cbc8511be9a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 14:45:23 +0200 Subject: fixed testbench compilation problem on Solaris Solaris network libraries needed to be specified in linker options --- tests/Makefile.am | 4 +++- tests/manytcp.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 6d3c8725..87dca985 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -33,9 +33,11 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg.sh ourtail_SOURCES = ourtail.c -tcpflood_SOURCES = tcpflood.c chkseq_SOURCES = chkseq.c +tcpflood_SOURCES = tcpflood.c +tcpflood_LDADD = $(SOL_LIBS) + nettester_SOURCES = nettester.c getline.c nettester_LDADD = $(SOL_LIBS) diff --git a/tests/manytcp.sh b/tests/manytcp.sh index b38245df..f0a3eb96 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,5 +1,6 @@ rm -f rsyslog.out.log # work file ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & +sleep 1 echo "rsyslogd started with pid " `cat rsyslog.pid` ./tcpflood 127.0.0.1 13514 1000 20000 sleep 1 -- cgit From e07b3f380f87166ce055ac4f3253f0367ef5cdce Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 18:51:21 +0200 Subject: working some more on "make distcheck" ... this time I think successfully (at least on Fedora...) --- Makefile.am | 2 +- tests/Makefile.am | 1 + tests/manytcp.sh | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index dfb33339..58e1cfce 100644 --- a/Makefile.am +++ b/Makefile.am @@ -127,5 +127,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-oracle +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout ACLOCAL_AMFLAGS = -I m4 diff --git a/tests/Makefile.am b/tests/Makefile.am index 53a81a93..6d3c8725 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -28,6 +28,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.omod-if-array \ parsertest.sh \ manytcp.sh \ + testsuites/manytcp.conf \ omod-if-array.sh \ cfg.sh diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 3accfb8a..b38245df 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,5 +1,5 @@ rm -f rsyslog.out.log # work file -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -ftestsuites/manytcp.conf & +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & echo "rsyslogd started with pid " `cat rsyslog.pid` ./tcpflood 127.0.0.1 13514 1000 20000 sleep 1 @@ -8,6 +8,8 @@ rm -f work sort < rsyslog.out.log > work ./chkseq work 0 19999 if [ "$?" -ne "0" ]; then + rm -f work rsyslog.out.log echo "sequence error detected" exit 1 fi +rm -f work rsyslog.out.log -- cgit From 9633ce1afcb3c3e44f969be0503360a4c0c75599 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 19:17:35 +0200 Subject: fixed compile-time problems in im3195 ... however, I did not not a test run due to the lack of existing test drivers and the very low (aka "non-existing" interest from the userbase in the feature). --- plugins/im3195/im3195.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/im3195/im3195.c b/plugins/im3195/im3195.c index 1c2502fe..106da2c8 100644 --- a/plugins/im3195/im3195.c +++ b/plugins/im3195/im3195.c @@ -47,6 +47,7 @@ #include "liblogging/syslogmessage.h" #include "module-template.h" #include "cfsysline.h" +#include "msg.h" #include "errmsg.h" MODULE_TYPE_INPUT @@ -83,7 +84,7 @@ void OnReceive(srAPIObj __attribute__((unused)) *pMyAPI, srSLMGObj* pSLMG) srSLMGGetRawMSG(pSLMG, &pszRawMsg); parseAndSubmitMessage(fromHost, fromHostIP, pszRawMsg, strlen((char*)pszRawMsg), - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_FULL_DELAY, (uchar*)"im3195"); + PARSE_HOSTNAME, eFLOWCTL_FULL_DELAY, (uchar*)"im3195", NULL, 0); } -- cgit From b9b96fbfc66809532f4ed24d667947fa63ce68f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 19:38:17 +0200 Subject: removed MSG_NOSIGNAL & provided work-around as this send() option is not supported on Solaris. We now simply ignore SIGPIPE --- tests/tcpflood.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 254e9fd6..fcb68998 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -170,7 +170,7 @@ int sendMessages(void) else socknum = rand() % numConnections; lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); - lenSend = send(sockArray[socknum], buf, lenBuf, MSG_NOSIGNAL); + lenSend = send(sockArray[socknum], buf, lenBuf, 0); if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); @@ -250,8 +250,17 @@ tcpSend(char *buf, int lenBuf) int main(int argc, char *argv[]) { int ret = 0; + struct sigaction sigAct; static char buf[1024]; + /* on Solaris, we do not HAVE MSG_NOSIGNAL, so for this reason + * we block SIGPIPE (not an issue for this program) + */ + memset(&sigAct, 0, sizeof(sigAct)); + sigemptyset(&sigAct.sa_mask); + sigAct.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sigAct, NULL); + setvbuf(stdout, buf, _IONBF, 48); if(argc != 5) { -- cgit From 8d65a9cdd49b812fb86cfd1b33f01c27e995db36 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 14:55:37 +0200 Subject: updated project status --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index dae94884..fff3a6ff 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,15 +5,15 @@

    This page reflects the status as of 2009-04-03.

    Current Releases

    -

    development: 4.1.5 [2009-03-11] - -change log - -download +

    development: 4.1.6 [2009-04-07] - +change log - +download
    beta: 3.21.11 [2009-04-03] - change log - download

    -

    v3 stable: 3.20.3 [2009-04-02] - change log - +

    v3 stable: 3.20.5 [2009-04-02] - change log - download
    v2 stable: 2.0.6 [2008-08-07] - change log - -- cgit From 04272876d12488b2039b28683dc53e1c802d303d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 13:52:07 +0200 Subject: implemented $MaxOpenFiles directive and changed testbench ... to utilize it. This work is not yet fully verified to be correct. --- runtime/rsyslog.h | 1 + tests/manytcp.sh | 13 +++++++--- tests/tcpflood.c | 59 +++++++++++++++++++++++++++++++++++-------- tests/testsuites/manytcp.conf | 4 ++- tools/syslogd.c | 28 ++++++++++++++++++++ 5 files changed, 89 insertions(+), 16 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index cea457d8..25ec30fc 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -262,6 +262,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ + RS_RET_ERR_RLIM_NOFILE = -2116, /**< error setting max. nbr open files process limit */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/tests/manytcp.sh b/tests/manytcp.sh index f0a3eb96..d9b2e9a0 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,13 +1,18 @@ -rm -f rsyslog.out.log # work file +rm -f work rsyslog.out.log rsyslog.out.log.save # work files ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & sleep 1 echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13514 1000 20000 -sleep 1 +# the config file specifies exactly 1100 connections +./tcpflood 127.0.0.1 13514 1000 40000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 5 # we need this so that rsyslogd can receive all outstanding messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 19999 +./chkseq work 0 39999 if [ "$?" -ne "0" ]; then rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 83f0d1ee..254e9fd6 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -95,21 +95,49 @@ int openConnections(void) sockArray = calloc(numConnections, sizeof(int)); for(i = 0 ; i < numConnections ; ++i) { if(i % 10 == 0) { - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d", i); + //lenMsg = sprintf(msgBuf, "\r%5.5d", i); + //write(1, msgBuf, lenMsg); } if(openConn(&(sockArray[i])) != 0) { printf("error in trying to open connection i=%d\n", i); return 1; } } - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d open connections\n", i); + lenMsg = sprintf(msgBuf, "\r%5.5d open connections\n", i); write(1, msgBuf, lenMsg); return 0; } +/* we also close all connections because otherwise we may get very bad + * timing for the syslogd - it may not be able to process all incoming + * messages fast enough if we immediately shut down. + * TODO: it may be an interesting excercise to handle that situation + * at the syslogd level, too + * rgerhards, 2009-04-14 + */ +void closeConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " close connections", sizeof(" close connections")-1); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + lenMsg = sprintf(msgBuf, "\r%5.5d", i); + write(1, msgBuf, lenMsg); + } + close(sockArray[i]); + } + lenMsg = sprintf(msgBuf, "\r%5.5d close connections\n", i); + write(1, msgBuf, lenMsg); + +} + + /* send messages to the tcp connections we keep open. We use * a very basic format that helps identify the message * (via msgnum:: e.g. msgnum:00000001:). This format is suitable @@ -123,13 +151,16 @@ int sendMessages(void) int i; int socknum; int lenBuf; + int lenSend; char buf[2048]; char msgBuf[128]; size_t lenMsg; srand(time(NULL)); /* seed is good enough for our needs */ - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent", 0); + printf("Sending %d messages.\n", numMsgsToSend); + printf("\r%5.5d messages sent", 0); + lenMsg = sprintf(msgBuf, "\r%5.5d/%5.5d messages sent", 0, numMsgsToSend); write(1, msgBuf, lenMsg); for(i = 0 ; i < numMsgsToSend ; ++i) { if(i < numConnections) @@ -139,18 +170,20 @@ int sendMessages(void) else socknum = rand() % numConnections; lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); - if(send(sockArray[socknum], buf, lenBuf, 0) != lenBuf) { + lenSend = send(sockArray[socknum], buf, lenBuf, MSG_NOSIGNAL); + if(lenSend != lenBuf) { + printf("\r%5.5d\n", i); + fflush(stdout); perror("send test data"); - fprintf(stderr, "send() failed\n"); + printf("send() failed at socket %d, index %d\n", socknum, i); + fflush(stderr); return(1); } if(i % 100 == 0) { - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d", i); } } - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent\n", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d messages sent\n", i); return 0; } @@ -219,7 +252,7 @@ int main(int argc, char *argv[]) int ret = 0; static char buf[1024]; - setvbuf(stdout, _IONBF, buf, 48); + setvbuf(stdout, buf, _IONBF, 48); if(argc != 5) { printf("Invalid call of tcpflood\n"); @@ -241,5 +274,9 @@ int main(int argc, char *argv[]) printf("error sending messages\n"); exit(1); } + + //closeConnections(); + printf("End of tcpflood Run\n"); + exit(ret); } diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index e491cd04..8175732e 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,7 +1,9 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 $ModLoad ../plugins/imtcp/.libs/imtcp -$inputtcpmaxsessions 2000 +$MainMsgQueueTimeoutShutdown 10000 +$MaxOpenFiles 2000 +$InputTCPMaxSessions 1100 $InputTCPServerRun 13514 $ErrorMessagesToStderr off diff --git a/tools/syslogd.c b/tools/syslogd.c index a4f0059b..8c86c12e 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -87,6 +87,7 @@ #include #include #include +#include #include #if HAVE_SYS_TIMESPEC_H @@ -2073,6 +2074,32 @@ static rsRetVal setActionResumeInterval(void __attribute__((unused)) *pVal, int } +/* set the processes max number ob files (upon configuration request) + * 2009-04-14 rgerhards + */ +static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) +{ + struct rlimit maxFiles; + char errStr[1024]; + DEFiRet; + + maxFiles.rlim_cur = iFiles; + maxFiles.rlim_max = iFiles; + + if(setrlimit(RLIMIT_NOFILE, &maxFiles) < 0) { + /* NOTE: under valgrind, we seem to be unable to extend the size! */ + rs_strerror_r(errno, errStr, sizeof(errStr)); + errmsg.LogError(0, RS_RET_ERR_RLIM_NOFILE, "could not set process file limit to %d: %s [kernel max %ld]", + iFiles, errStr, (long) maxFiles.rlim_max); + ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); + } + dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); + +finalize_it: + RETiRet; +} + + /* set the processes umask (upon configuration request) */ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { @@ -2870,6 +2897,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"modload", 0, eCmdHdlrCustomHandler, conf.doModLoad, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"includeconfig", 0, eCmdHdlrCustomHandler, conf.doIncludeLine, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"umask", 0, eCmdHdlrFileCreateMode, setUmask, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"maxopenfiles", 0, eCmdHdlrInt, setMaxFiles, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprinttemplatelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintTemplateList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintmodulelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintModuleList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintcfsyslinehandlerlist", 0, eCmdHdlrBinary, -- cgit From 01e5d51c57536acfa8e05d5b22c001a6cf0701d8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 14:36:50 +0200 Subject: added doc for $MaxOpenFiles directive --- ChangeLog | 1 + doc/rsconf1_maxopenfiles.html | 35 +++++++++++++++++++++++++++++++++++ doc/rsyslog_conf.html | 6 +++--- doc/rsyslog_conf_global.html | 18 +++++++++--------- 4 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 doc/rsconf1_maxopenfiles.html diff --git a/ChangeLog b/ChangeLog index 792c108c..3108cdb3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? - improved testbench * added tests for tcp-based reception * added tcp-load test (1000 connections, 20,000 messages) +- added $MaxOpenFiles configuration directive - bugfix: solved potential memory leak in msg processing, could manifest itself in imtcp --------------------------------------------------------------------------- diff --git a/doc/rsconf1_maxopenfiles.html b/doc/rsconf1_maxopenfiles.html new file mode 100644 index 00000000..b6c9cc0e --- /dev/null +++ b/doc/rsconf1_maxopenfiles.html @@ -0,0 +1,35 @@ + + +$MaxOpenFiles - rsyslog.conf file + + +[rsyslog configuration directive overview] + +

    $MaxOpenFiles

    +

    Available Since: 4.3.0

    +

    Type: global configuration directive

    +

    Default: operating system default

    +

    Description:

    +

    Set the maximum number of files that the rsyslog process can have open at any given +time. Note that this includes open tcp sockets, so this setting is the upper limit for +the number of open TCP connections as well. If you expect a large nubmer of concurrent +connections, it is suggested that the number is set to the max number connected plus 1000. +Please note that each dynafile also requires up to 100 open file handles. +

    The setting is similar to running "ulimit -n number-of-files". +

    Please note that depending on permissions and operating system configuration, the +setrlimit() request issued by rsyslog may fail, in which case the previous limit is kept +in effect. Rsyslog will emit a warning message in this case. +

    Sample:

    +

    $MaxOpenFiles 2000

    +

    Bugs:

    +

    For some reason, this settings seems not to work on all platforms. If you experience +problems, please let us know so that we can (hopefully) narrow down the issue. +

    [rsyslog.conf overview] [manual +index] [rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 3 or higher.

    + + diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 852d95b5..6990c6bd 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -26,7 +26,7 @@ Lines can be continued by specifying a backslash ("\") as the last character of the line. There is a hard-coded maximum line length of 4K. If you need lines larger than that, you need to change compile-time settings inside rsyslog and recompile. -

    Global Directives

    +

    Configuration Directives

    Basic Structure

    Rsyslog supports standard sysklogd's configuration file format and extends it. So in general, you can take a "normal" syslog.conf and @@ -74,9 +74,9 @@ such features is available in rsyslogd, only.

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008,2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    > diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d011bd2b..3e33f0da 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -1,14 +1,14 @@ -Global Directives - rsyslog.conf +Configuration Directives - rsyslog.conf

    This is a part of the rsyslog.conf documentation.

    back -

    Global Directives

    -

    All global directives need to be specified on a line by their -own and must start with a dollar-sign. Here is a list in alphabetical -order. Follow links for a description.

    -

    Please note that not all directives here are actually global. Some affect -only the next action. This documentation will be changed soon. +

    Configuration Directives

    +

    All configuration directives need to be specified on a line by their +own and must start with a dollar-sign. Note that those starting with +the word "Action" modify the next action and should be specified +in front of it. +

    Here is a list in alphabetical order. Follow links for a description.

    Not all directives have an in-depth description right now. Default values for them are in bold. A more in-depth description will appear as implementation progresses. @@ -180,6 +180,7 @@ instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restri
    Note that 2k, the current default, is the smallest size that must be supported in order to be compliant to the upcoming new syslog RFC series.

    • +
  • $MaxOpenFiles
  • $ModDir
  • $ModLoad
  • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, @@ -214,7 +215,6 @@ the value, the less precise the timestamp.
  • $PrivDropToGroupID
  • $PrivDropToUser
  • $PrivDropToUserID
    • -
  • $UMASK

    • Where <size_nbr> is specified above, @@ -235,7 +235,7 @@ point of view, "1,,0.0.,.,0" also has the value 1000.

    rsyslog project.
    Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    -- cgit From ba5a59128f6559d58cdd4defe46a9db564d3e2c1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 20:48:08 +0200 Subject: cosmetic fix (status message) --- tests/tcpflood.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index fcb68998..9c17fd5b 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -160,8 +160,6 @@ int sendMessages(void) printf("Sending %d messages.\n", numMsgsToSend); printf("\r%5.5d messages sent", 0); - lenMsg = sprintf(msgBuf, "\r%5.5d/%5.5d messages sent", 0, numMsgsToSend); - write(1, msgBuf, lenMsg); for(i = 0 ; i < numMsgsToSend ; ++i) { if(i < numConnections) socknum = i; -- cgit From 65a85de3d97ab6bc427ea005b75e4b416013de3c Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:16 +0200 Subject: Convert to the array-based interface. We'll receive a single statement to be prepared and a batch size. Then, doAction will execute the statement only once per batch hit, making the process much more efficient. This will reduce network and DB server overhead. The downside is that this version cannot be used with rsyslog v3 anymore. If anyone is interested on backporting the module, they should choose all patches up to this one. Better documentation may follow. --- plugins/omoracle/omoracle.c | 203 +++++++++++++++++++++++++++++++++++--------- plugins/omoracle/omoracle.h | 2 + 2 files changed, 164 insertions(+), 41 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index f6679953..02f83551 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -21,9 +21,28 @@ $OmoracleBatchSize: Number of elements to send to the DB on each transaction. + $OmoracleStatement: Statement to be prepared and executed in + batches. Please note that Oracle's prepared statements have their + placeholders as ':identifier', and this module uses the colon to + guess how many placeholders there will be. + All these directives are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. + The form of the template is just a list of strings you want + inserted to the DB, for instance: + + $template TestStmt,"%hostname%%msg%" + + Will provide the arguments to a statement like + + $OmoracleStatement \ + insert into foo(hostname,message)values(:host,:message) + + Also note that identifiers to placeholders are arbitrarry. You + need to define the properties on the template in the correct order + you want them passed to the statement! + Author: Luis Fernando Muñoz Mejías @@ -40,6 +59,7 @@ #include #include #include +#include #include "dirty.h" #include "syslogd-types.h" #include "srUtils.h" @@ -63,8 +83,12 @@ struct oracle_batch /* Last element inserted in the buffer. The batch will be * executed when n == size */ int n; - /* Statements to run on this transaction */ - char** statements; + /* Number of arguments the statement takes */ + int arguments; + /* Parameters to pass to the statement on this transaction */ + char*** parameters; + /* Binding parameters */ + OCIBind** bindings; }; typedef struct _instanceData { @@ -80,10 +104,10 @@ typedef struct _instanceData { OCISvcCtx* service; /* Credentials object for the connection. */ OCIAuthInfo* authinfo; - /* Binding parameters, currently unused */ - OCIBind* binding; /* Connection string, kept here for possible retries. */ char* connection; + /* Statement to be prepared. */ + char* txt_statement; /* Batch */ struct oracle_batch batch; } instanceData; @@ -97,6 +121,12 @@ static char* db_user; static char* db_password; /** Batch size. */ static int batch_size; +/** Statement to prepare and execute */ +static char* db_statement; +/** Whether or not the core supports the newer array interface. The + * module is able to work in both modes, but the newer is the + * recommended one for performance reasons. */ +static int array_passing; /** Generic function for handling errors from OCI. @@ -149,9 +179,49 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_ERROR; } +/** Returns the number of bind parameters for the statement given as + * an argument. */ +static int count_bind_parameters(char* p) +{ + int n = 0; + + for (; *p; p++) + if (*p == BIND_MARK) + n++; + dbgprintf ("omoracle statement has %d parameters\n", n); + return n; +} + +/** Prepares the statement, binding all its positional parameters */ +static int prepare_statement(instanceData* pData) +{ + int i; + DEFiRet; + + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->txt_statement, + strlen(pData->txt_statement), + OCI_NTV_SYNTAX, OCI_DEFAULT)); + for (i = 0; i < pData->batch.arguments; i++) + CHECKERR(pData->error, + OCIBindByPos(pData->statement, + pData->batch.bindings+i, + pData->error, + i+1, + pData->batch.parameters[i], + sizeof (OCILobLocator*), + SQLT_STR, NULL, NULL, NULL, + 0, 0, OCI_DEFAULT)); +finalize_it: + RETiRet; +} + /* Resource allocation */ BEGINcreateInstance + int i; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -168,12 +238,29 @@ CODESTARTcreateInstance CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); + pData->txt_statement = strdup(db_statement); + CHKmalloc(pData->txt_statement); + dbgprintf("omoracle will run stored statement: %s\n", + pData->txt_statement); pData->batch.n = 0; pData->batch.size = batch_size; - pData->batch.statements = calloc(pData->batch.size, - sizeof *pData->batch.statements); - CHKmalloc(pData->batch.statements); + pData->batch.arguments = count_bind_parameters(pData->txt_statement); + + /* I know, this can be done with a single malloc() call but this is + * easier to read. :) */ + pData->batch.parameters = malloc(pData->batch.arguments * + sizeof *pData->batch.parameters); + CHKmalloc(pData->batch.parameters); + for (i = 0; i < pData->batch.arguments; i++) { + pData->batch.parameters[i] = calloc(pData->batch.size, + sizeof **pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i]); + } + + pData->batch.bindings = calloc(pData->batch.arguments, + sizeof *pData->batch.bindings); + CHKmalloc(pData->batch.bindings); finalize_it: ENDcreateInstance @@ -183,35 +270,34 @@ ENDcreateInstance static int insert_to_db(instanceData* pData) { DEFiRet; - int i, n; + int i, j, n; + + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + pData->batch.n, 0, NULL, NULL, OCI_DEFAULT)); - for (i = 0; i < pData->batch.n; i++) { - if (pData->batch.statements[i] == NULL) - continue; - n = strlen(pData->batch.statements[i]); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, - pData->error, - pData->batch.statements[i], n, - OCI_NTV_SYNTAX, OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, - pData->statement, - pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - free(pData->batch.statements[i]); - pData->batch.statements[i] = NULL; - } CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); + + for (i = 0; i < pData->batch.arguments; i++) + for (j = 0; j < pData->batch.n; j++) { + free(pData->batch.parameters[i][j]); + pData->batch.parameters[i][j] = NULL; + } + pData->batch.n = 0; finalize_it: + dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? + "succeeded" : "did not succeed"); RETiRet; } /** Close the session and free anything allocated by createInstance. */ BEGINfreeInstance + int i, j; CODESTARTfreeInstance /* Before actually releasing our resources, let's try to commit @@ -224,9 +310,13 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); - while (pData->batch.size--) - free(pData->batch.statements[pData->batch.size]); - free(pData->batch.statements); + for (i = 0; i < pData->batch.arguments; i++) { + for (j = 0; j < pData->batch.size; j++) + free(pData->batch.parameters[i][j]); + free(pData->batch.parameters[i]); + } + free(pData->batch.parameters); + free(pData->batch.bindings); dbgprintf ("omoracle freed all its resources\n"); ENDfreeInstance @@ -253,7 +343,7 @@ CODESTARTtryResume * ... of course I don't know why Oracle might need a full restart... * rgerhards, 2009-03-26 */ - dbgprintf("Attempting to reconnect to DB server\n"); + dbgprintf("omoracle attempting to reconnect to DB server\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, @@ -261,6 +351,7 @@ CODESTARTtryResume pData->connection, strlen(pData->connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); + CHKiRet(prepare_statement(pData)); finalize_it: ENDtryResume @@ -296,7 +387,6 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature BEGINparseSelectorAct - CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -314,11 +404,12 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); } CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, - OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + OMSR_TPL_AS_ARRAY, " StdFmt")); CHKiRet(createInstance(&pData)); CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); - + CHKiRet(prepare_statement(pData)); + dbgprintf ("omoracle module got all its resources allocated " "and connected to the DB\n"); @@ -327,21 +418,23 @@ ENDparseSelectorAct BEGINdoAction int i; - int n; + int n = pData->batch.n; + char **params = (char**) ppString[0]; CODESTARTdoAction - dbgprintf("omoracle attempting to execute statement %s\n", *ppString); - if (pData->batch.n == pData->batch.size) { + if (n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); CHKiRet(insert_to_db(pData)); } - pData->batch.statements[pData->batch.n] = strdup(*ppString); - CHKmalloc(pData->batch.statements[pData->batch.n]); + + for (i = 0; i < pData->batch.arguments && params[i]; i++) { + dbgprintf ("omoracle argument on batch[%d][%d]: %s\n", i, n, params[i]); + pData->batch.parameters[i][n] = strdup(params[i]); + CHKmalloc(pData->batch.parameters[i][n]); + } pData->batch.n++; finalize_it: - dbgprintf ("omoracle %s at executing statement %s\n", - iRet?"did not succeed":"succeeded", *ppString); ENDdoAction BEGINmodExit @@ -373,16 +466,35 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, memset(db_password, 0, n); free(db_password); } - db_name = db_user = db_password = NULL; + if (db_statement != NULL) + free(db_statement); + db_name = db_user = db_password = db_statement = NULL; + RETiRet; +} + +/** As I don't find any handler that reads an entire line, I write my + * own. */ +static int get_db_statement(char** line, char** stmt) +{ + DEFiRet; + + while (isspace(**line)) + (*line)++; + dbgprintf ("Config line: %s\n", *line); + *stmt = strdup(*line); + CHKmalloc(*stmt); + dbgprintf ("Statement: %s\n", *stmt); +finalize_it: RETiRet; } BEGINmodInit() + rsRetVal (*supported_options)(unsigned long *pOpts); + unsigned long opts; CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); - /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); @@ -398,4 +510,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchsize", 0, eCmdHdlrInt, NULL, &batch_size, STD_LOADABLE_MODULE_ID)); + CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); + CHKiRet((*supported_options)(&opts)); + if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) + ABORT_FINALIZE(RS_RET_ERR); + + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, + eCmdHdlrCustomHandler, get_db_statement, + &db_statement, STD_LOADABLE_MODULE_ID)); + ENDmodInit diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h index b0e70917..92bcacab 100644 --- a/plugins/omoracle/omoracle.h +++ b/plugins/omoracle/omoracle.h @@ -20,4 +20,6 @@ enum { MAX_BUFSIZE = 2048 }; +#define BIND_MARK ':' + #endif -- cgit From 24fcd96203c9b8b84a8414cea19dcb3ba989c9ba Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:17 +0200 Subject: Fixed a mem leak --- plugins/omoracle/omoracle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 02f83551..12efd61c 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -310,6 +310,7 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); + free(pData->txt_statement); for (i = 0; i < pData->batch.arguments; i++) { for (j = 0; j < pData->batch.size; j++) free(pData->batch.parameters[i][j]); -- cgit From f89b761c84270cde71e0a6275ea80bb20f60d2df Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:18 +0200 Subject: Make the counting of bind parameters aware of literals. Literal strings passed in the statement may contain ':', let's not count them. --- plugins/omoracle/omoracle.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 12efd61c..b598192f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -180,14 +180,21 @@ static int oci_errors(void* handle, ub4 htype, sword status) } /** Returns the number of bind parameters for the statement given as - * an argument. */ + * an argument. It counts the number of appearances of ':', as in + * + * insert into foo(bar, baz) values(:bar, :baz) + * + * while taking in account that string literals must not be parsed. */ static int count_bind_parameters(char* p) { int n = 0; + int enable = 1; for (; *p; p++) - if (*p == BIND_MARK) + if (enable && *p == BIND_MARK ) n++; + else if (*p == '\'') + enable ^= 1; dbgprintf ("omoracle statement has %d parameters\n", n); return n; } @@ -429,7 +436,6 @@ CODESTARTdoAction } for (i = 0; i < pData->batch.arguments && params[i]; i++) { - dbgprintf ("omoracle argument on batch[%d][%d]: %s\n", i, n, params[i]); pData->batch.parameters[i][n] = strdup(params[i]); CHKmalloc(pData->batch.parameters[i][n]); } -- cgit From 9a897329ec6f80c99ca039f12388961417e0a422 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:19 +0200 Subject: Add some debugging output --- plugins/omoracle/omoracle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index b598192f..7199f3e1 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -436,6 +436,7 @@ CODESTARTdoAction } for (i = 0; i < pData->batch.arguments && params[i]; i++) { + dbgprintf("batch[%d][%d]=%s\n", i, n, params[i]); pData->batch.parameters[i][n] = strdup(params[i]); CHKmalloc(pData->batch.parameters[i][n]); } -- cgit From ca28204f7ba5c5c520b52180e26471e12af83560 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:20 +0200 Subject: Add the callback for OCIBindDynamic. Let's hope it works. --- plugins/omoracle/omoracle.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 7199f3e1..48b97b27 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -179,6 +179,34 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_ERROR; } +/** Callback for OCIBindDynamic. + * + * OCI doesn't insert an array of char* by itself (although it can + * handle arrays of int), so we must either run in batches of size one + * (no way) or bind all parameters with OCI_DATA_AT_EXEC instead of + * OCI_DEFAULT, and then give this function as an argument to + * OCIBindDynamic so that it is able to handle all strings in a single + * server trip. + * + * See the documentation of OCIBindDynamic + * (http://download.oracle.com/docs/cd/B28359_01/appdev.111/b28395/oci16rel003.htm#i444015) + * for more details. + */ +static int __attribute__((unused)) +bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, + int iter, int __attribute__((unused)) idx, + char** out, int* buflen, char* piece, + void** bd) +{ + dbgprintf ("Bound line: %s\n", in[iter]); + *out = in[iter]; + *buflen = sizeof (OCILobLocator*); + *piece = OCI_ONE_PIECE; + *bd = NULL; + return OCI_CONTINUE; +} + + /** Returns the number of bind parameters for the statement given as * an argument. It counts the number of appearances of ':', as in * -- cgit From 668f9a79fb7268f7935d93249cf283664662996d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:21 +0200 Subject: Fixing the batch insertions. Previous versions inserted garbage (the pointer was interpreted as the string itself). It seems inserting arrays of strings is not that easy with OCI. This approach consumes 2KB per entry in the batch, so if you have batches of size 1000 you'll be using 2MB for the batch. This size doesn't change, anyways and the risk of leaking memory is gone. OCI doesn't deal well with batches of strings. :( --- plugins/omoracle/omoracle.c | 63 ++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 48b97b27..ddcb2ffa 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -192,15 +192,15 @@ static int oci_errors(void* handle, ub4 htype, sword status) * (http://download.oracle.com/docs/cd/B28359_01/appdev.111/b28395/oci16rel003.htm#i444015) * for more details. */ -static int __attribute__((unused)) -bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, - int iter, int __attribute__((unused)) idx, - char** out, int* buflen, char* piece, +static int bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, + int iter, int __attribute__((unused)) idx, + char** out, int* buflen, unsigned char* piece, void** bd) { - dbgprintf ("Bound line: %s\n", in[iter]); *out = in[iter]; - *buflen = sizeof (OCILobLocator*); + *buflen = strlen(*out) + 1; + dbgprintf ("omoracle bound line %d, length %d: %s\n", iter, *buflen, + *out); *piece = OCI_ONE_PIECE; *bd = NULL; return OCI_CONTINUE; @@ -239,16 +239,22 @@ static int prepare_statement(instanceData* pData) pData->txt_statement, strlen(pData->txt_statement), OCI_NTV_SYNTAX, OCI_DEFAULT)); - for (i = 0; i < pData->batch.arguments; i++) + for (i = 0; i < pData->batch.arguments; i++) { CHECKERR(pData->error, OCIBindByPos(pData->statement, pData->batch.bindings+i, - pData->error, - i+1, - pData->batch.parameters[i], - sizeof (OCILobLocator*), + pData->error, i+1, NULL, + MAX_BUFSIZE * + sizeof ***pData->batch.parameters, SQLT_STR, NULL, NULL, NULL, - 0, 0, OCI_DEFAULT)); + 0, 0, OCI_DATA_AT_EXEC)); + CHECKERR(pData->error, + OCIBindDynamic(pData->batch.bindings[i], + pData->error, + pData->batch.parameters[i], + bind_dynamic, NULL, NULL)); + } + finalize_it: RETiRet; } @@ -256,7 +262,7 @@ finalize_it: /* Resource allocation */ BEGINcreateInstance - int i; + int i, j; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -284,13 +290,25 @@ CODESTARTcreateInstance /* I know, this can be done with a single malloc() call but this is * easier to read. :) */ - pData->batch.parameters = malloc(pData->batch.arguments * + pData->batch.parameters = calloc(pData->batch.arguments, sizeof *pData->batch.parameters); CHKmalloc(pData->batch.parameters); for (i = 0; i < pData->batch.arguments; i++) { pData->batch.parameters[i] = calloc(pData->batch.size, sizeof **pData->batch.parameters); CHKmalloc(pData->batch.parameters[i]); + for (j = 0; j < pData->batch.size; j++) { + /* Each entry has at most MAX_BUFSIZE bytes + * because OCI doesn't like null-terminated + * strings when operating with batches, and + * the maximum size of each entry must be + * provided when binding + * parameters. MAX_BUFSIZE is long enough for + * usual entries. */ + pData->batch.parameters[i][j] = calloc(MAX_BUFSIZE, + sizeof ***pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i][j]); + } } pData->batch.bindings = calloc(pData->batch.arguments, @@ -305,7 +323,6 @@ ENDcreateInstance static int insert_to_db(instanceData* pData) { DEFiRet; - int i, j, n; CHECKERR(pData->error, OCIStmtExecute(pData->service, @@ -316,12 +333,6 @@ static int insert_to_db(instanceData* pData) CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); - for (i = 0; i < pData->batch.arguments; i++) - for (j = 0; j < pData->batch.n; j++) { - free(pData->batch.parameters[i][j]); - pData->batch.parameters[i][j] = NULL; - } - pData->batch.n = 0; finalize_it: dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? @@ -454,19 +465,19 @@ ENDparseSelectorAct BEGINdoAction int i; - int n = pData->batch.n; char **params = (char**) ppString[0]; CODESTARTdoAction - if (n == pData->batch.size) { + if (pData->batch.n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); CHKiRet(insert_to_db(pData)); } for (i = 0; i < pData->batch.arguments && params[i]; i++) { - dbgprintf("batch[%d][%d]=%s\n", i, n, params[i]); - pData->batch.parameters[i][n] = strdup(params[i]); - CHKmalloc(pData->batch.parameters[i][n]); + dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); + strncpy(pData->batch.parameters[i][pData->batch.n], params[i], + MAX_BUFSIZE); + CHKmalloc(pData->batch.parameters[i][pData->batch.n]); } pData->batch.n++; -- cgit From 3c886026be31a6af61a1b86773634c7bc4a44d7e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 16:55:25 +0200 Subject: added some doc for omoracle --- doc/Makefile.am | 1 + doc/modules.html | 5 ++- doc/omoracle.html | 78 +++++++++++++++++++++++++++++++++++++++++++ doc/rsyslog_conf_modules.html | 5 +-- 4 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 doc/omoracle.html diff --git a/doc/Makefile.am b/doc/Makefile.am index 3015d6b5..0f3dca70 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -33,6 +33,7 @@ html_files = \ dev_queue.html \ omsnmp.html \ ommysql.html \ + omoracle.html \ omlibdbi.html \ imfile.html \ imtcp.html \ diff --git a/doc/modules.html b/doc/modules.html index 92887508..4eae6db3 100644 --- a/doc/modules.html +++ b/doc/modules.html @@ -4,9 +4,8 @@

    About rsyslog Modules

    -

    Written by - Rainer - Gerhards (2007-07-28)

    +

    Written by +Rainer Gerhards (2007-07-28)

    This document is incomplete. The module interface is also quite incomplete and under development. Do not currently use it! You may want to visit Rainer's blog diff --git a/doc/omoracle.html b/doc/omoracle.html new file mode 100644 index 00000000..40f6360f --- /dev/null +++ b/doc/omoracle.html @@ -0,0 +1,78 @@ + + + +Oracle Database Output Module + + + +rsyslog module reference + +

    Oracle Database Output Module

    +

    Module Name:    omoracle

    +

    Author: Luis Fernando Muñoz Mejías <Luis.Fernando.Munoz.Mejias@cern.ch>

    +

    Available since: : 4.3.0 +

    Status: : contributed module, not maitained by rsyslog core authors +

    Description:

    +

    This module provides native support for logging to Oracle databases. It offers +superior performance over the more generic omlibdbi module. +It also includes a number of enhancements, most importantly prepared statements and +batching, what provides a big performance improvements. +

    +

    Note that this module is maintained by its original author. If you need assistance with it, +it is suggested to post questions to the +rsyslog mailing list. +

    From the header comments of this module: +

    +
+    This is an output module feeding directly to an Oracle
+    database. It uses Oracle Call Interface, a propietary module
+    provided by Oracle.
+
+    Selector lines to be used are of this form:
+
+    :omoracle:;TemplateName
+
+    The module gets its configuration via rsyslog $... directives,
+    namely:
+
+    $OmoracleDBUser: user name to log in on the database.
+
+    $OmoracleDBPassword: password to log in on the database.
+
+    $OmoracleDB: connection string (an Oracle easy connect or a db
+    name as specified by tnsnames.ora)
+
+    $OmoracleBatchSize: Number of elements to send to the DB on each
+    transaction.
+
+    $OmoracleStatement: Statement to be prepared and executed in
+    batches. Please note that Oracle's prepared statements have their
+    placeholders as ':identifier', and this module uses the colon to
+    guess how many placeholders there will be.
+
+    All these directives are mandatory. The dbstring can be an Oracle
+    easystring or a DB name, as present in the tnsnames.ora file.
+
+    The form of the template is just a list of strings you want
+    inserted to the DB, for instance:
+
+    $template TestStmt,"%hostname%%msg%"
+
+    Will provide the arguments to a statement like
+
+    $OmoracleStatement \
+        insert into foo(hostname,message)values(:host,:message)
+
+    Also note that identifiers to placeholders are arbitrarry. You
+    need to define the properties on the template in the correct order
+    you want them passed to the statement!
+
    +

    [rsyslog.conf overview] +[manual index] [rsyslog site]

    +

    This documentation is part of the +rsyslog +project.
    +Copyright © 2008, 2009 by Rainer Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

    + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index a281d9e7..df9abeea 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -19,6 +19,7 @@ generic database output module (Firebird/Interbase, MS SQL, Sybase, SQLLite, Ingres, Oracle, mSQL)
  • ommail - permits rsyslog to alert folks by mail if something important happens
    • +
  • omoracle - output module for Oracle (native OCI interface)
  • imfile -  input module for text files
  • imrelp - RELP @@ -44,9 +45,9 @@ only available if it has been loaded (using $ModLoad).

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    -- cgit From 2d5e8ba7cd05a95f897506e51dcc5adb06dbcaa8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 17:26:07 +0200 Subject: added a new error code for too-old rsyslog core which can be emittend when plugin can not load due to missing core functionality. --- plugins/omoracle/omoracle.c | 2 +- runtime/rsyslog.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ddcb2ffa..71cc8e1f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -560,7 +560,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); CHKiRet((*supported_options)(&opts)); if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, eCmdHdlrCustomHandler, get_db_statement, diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index cea457d8..8e181b9e 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -262,6 +262,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ + RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From 9348f8b9c2b5ba806dab8c7337877ab9da3f85b1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:33:14 +0200 Subject: preparing for 4.3.0 release --- ChangeLog | 7 ++++++- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 90b351f5..d01f3cb0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? +Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program and feed it (via its stdin) with syslog messages. If the program terminates, it is restarted. @@ -14,6 +14,11 @@ Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? - added $MaxOpenFiles configuration directive - bugfix: solved potential memory leak in msg processing, could manifest itself in imtcp +- bugfix: ompgsql did not detect problems in sql command execution + this could cause loss of messages. The handling was correct if the + connection broke, but not if there was a problem with statement + execution. The most probable case for such a case would be invalid + sql inside the template, and this is now much easier to diagnose. --------------------------------------------------------------------------- Version 4.1.7 [BETA] (rgerhards), 2009-04-?? - bugfix: $InputTCPMaxSessions config directive was accepted, but not diff --git a/configure.ac b/configure.ac index c7f1d532..6daf3f5a 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.3.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index 51271701..ae90f780 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 4.1.6 (devel branch) of rsyslog. +

    This documentation is for version 4.3.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit From dc777849fcd5c300f229c44daf4278d8b6842b71 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:37:41 +0200 Subject: bugfix: missing header (platform compatibility issue) --- tests/tcpflood.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 9c17fd5b..8dbc201b 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include -- cgit From 09ca443377ec85364160e965920ab6f56a374d88 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 15:10:06 +0200 Subject: update project status & cleanup removed some warning in imklog compilation, but may not have solved a lurking issue (but placed comment so that we know if something surfaces) --- doc/status.html | 16 ++++++++-------- plugins/imklog/ksym_mod.c | 42 ++++++++++++++++++++++-------------------- plugins/imklog/ksyms.h | 4 ++-- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/doc/status.html b/doc/status.html index fff3a6ff..01a9cf8f 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,22 +2,22 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2009-04-03.

    +

    This page reflects the status as of 2009-04-17.

    Current Releases

    -

    development: 4.1.6 [2009-04-07] - -change log - -download +

    development: 4.3.0 [2009-04-17] - +change log - +download
    beta: 3.21.11 [2009-04-03] - change log - download

    -

    v3 stable: 3.20.5 [2009-04-02] - change log - -download +

    v3 stable: 3.20.6 [2009-04-16] - change log - +download -
    v2 stable: 2.0.6 [2008-08-07] - change log - -download +
    v2 stable: 2.0.7 [2009-04-14] - change log - +download
    v0 and v1 are deprecated and no longer supported. If you absolutely do not like to upgrade, you may consider purchasing a commercial rsyslog support package. Just let us point diff --git a/plugins/imklog/ksym_mod.c b/plugins/imklog/ksym_mod.c index 6e48e89e..be5fdee9 100644 --- a/plugins/imklog/ksym_mod.c +++ b/plugins/imklog/ksym_mod.c @@ -1,9 +1,8 @@ -/* - * ksym_mod.c - functions for building symbol lookup tables for klogd +/* ksym_mod.c - functions for building symbol lookup tables for klogd * Copyright (c) 1995, 1996 Dr. G.W. Wettstein * Copyright (c) 1996 Enjellic Systems Development * Copyright (c) 1998-2007 Martin Schulze - * Copyright (C) 2007-2008 Rainer Gerhards + * Copyright (C) 2007-2009 Rainer Gerhards * * This file is part of rsyslog. * @@ -83,7 +82,6 @@ * Changed llseek() to lseek64() in order to skip a libc warning. */ - /* Includes. */ #include "config.h" #include @@ -112,7 +110,7 @@ #define KSYMS "/proc/kallsyms" static int num_modules = 0; -struct Module *sym_array_modules = (struct Module *) 0; +struct Module *sym_array_modules = (struct Module *) NULL; static int have_modules = 0; @@ -266,7 +264,7 @@ static void FreeModules() } free(sym_array_modules); - sym_array_modules = (struct Module *) 0; + sym_array_modules = (struct Module *) NULL; num_modules = 0; return; } @@ -390,11 +388,11 @@ static int AddSymbol(line) mp->sym_array = (struct sym_table *) realloc(mp->sym_array, \ (mp->num_syms+1) * sizeof(struct sym_table)); - if ( mp->sym_array == (struct sym_table *) 0 ) + if ( mp->sym_array == (struct sym_table *) NULL ) return(0); mp->sym_array[mp->num_syms].name = strdup(p); - if ( mp->sym_array[mp->num_syms].name == (char *) 0 ) + if ( mp->sym_array[mp->num_syms].name == (char *) NULL ) return(0); /* Stuff interesting information into the module. */ @@ -424,15 +422,21 @@ static int AddSymbol(line) * If a match cannot be found a diagnostic string is printed. * If a match is found the pointer to the symbolic name most * closely matching the address is returned. + * + * TODO: We are using int values for the offset, but longs for the value + * values. This may create some trouble in the future (on 64 Bit OS?). + * Anyhow, I have not changed this, because we do not seem to have any + * issue and my understanding of this code is limited (and I don't see + * need to invest more time to dig much deeper). + * rgerhards, 2009-04-17 **************************************************************************/ extern char * LookupModuleSymbol(value, sym) unsigned long value; struct symbol *sym; { - auto int nmod, - nsym; - auto struct sym_table *last; - auto struct Module *mp; + int nmod, nsym; + struct sym_table *last; + struct Module *mp; static char ret[100]; sym->size = 0; @@ -443,8 +447,7 @@ extern char * LookupModuleSymbol(value, sym) for (nmod = 0; nmod < num_modules; ++nmod) { mp = &sym_array_modules[nmod]; - /* - * Run through the list of symbols in this module and + /* Run through the list of symbols in this module and * see if the address can be resolved. */ for(nsym = 1, last = &mp->sym_array[0]; @@ -453,13 +456,12 @@ extern char * LookupModuleSymbol(value, sym) if ( mp->sym_array[nsym].value > value ) { if ( sym->size == 0 || - (value - last->value) < sym->offset || - ( (sym->offset == (value - last->value)) && - (mp->sym_array[nsym].value-last->value) < sym->size ) ) + (int) (value - last->value) < sym->offset || + ( (sym->offset == (int) (value - last->value)) && + (int) (mp->sym_array[nsym].value-last->value) < sym->size ) ) { sym->offset = value - last->value; - sym->size = mp->sym_array[nsym].value - \ - last->value; + sym->size = mp->sym_array[nsym].value - last->value; ret[sizeof(ret)-1] = '\0'; if ( mp->name == NULL ) snprintf(ret, sizeof(ret)-1, @@ -478,5 +480,5 @@ extern char * LookupModuleSymbol(value, sym) return(ret); /* It has been a hopeless exercise. */ - return((char *) 0); + return(NULL); } diff --git a/plugins/imklog/ksyms.h b/plugins/imklog/ksyms.h index b5362ff3..a168947b 100644 --- a/plugins/imklog/ksyms.h +++ b/plugins/imklog/ksyms.h @@ -2,7 +2,7 @@ * Copyright (c) 1995, 1996 Dr. G.W. Wettstein * Copyright (c) 1996 Enjellic Systems Development * Copyright (c) 2004-7 Martin Schulze - * Copyright (c) 2007-2008 Rainer Gerhards + * Copyright (c) 2007-2009 Rainer Gerhards * * This file is part of rsyslog. * @@ -26,7 +26,7 @@ struct symbol { - char *name; + uchar *name; int size; int offset; }; -- cgit From 889a0a1da8b2fb74b04647a345f64fce6c36708f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 15:19:57 +0200 Subject: some cleanup ... mostly removal of compile-time warnings (thanks to Michael Biebl for suggesting to look after that) --- ChangeLog | 2 ++ plugins/imgssapi/imgssapi.c | 1 - plugins/omgssapi/omgssapi.c | 2 +- runtime/nsd_gtls.c | 2 ++ tools/msggen.c | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index d01f3cb0..48343b5f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ --------------------------------------------------------------------------- +Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +--------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program and feed it (via its stdin) with syslog messages. If the program diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index debe935e..b9d7dfe3 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -249,7 +249,6 @@ onErrClose(tcps_sess_t *pSess) static rsRetVal doOpenLstnSocks(tcpsrv_t *pSrv) { - int *pRet = NULL; gsssrv_t *pGSrv; DEFiRet; diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index e0cc8af6..361f657f 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -444,7 +444,7 @@ CODESTARTdoAction /* error! */ dbgprintf("error forwarding via tcp, suspending\n"); pData->eDestState = eDestFORW_SUSP; - iRet = RS_RET_SUSPENDED; + ABORT_FINALIZE(RS_RET_SUSPENDED); } break; } diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 3a79a015..5786e191 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -82,6 +82,7 @@ static gnutls_certificate_credentials xcred; static gnutls_dh_params dh_params; #ifdef DEBUG +#if 0 /* uncomment, if needed some time again -- DEV Debug only */ /* This defines a log function to be provided to GnuTLS. It hopefully * helps us track down hard to find problems. * rgerhards, 2008-06-20 @@ -90,6 +91,7 @@ static void logFunction(int level, const char *msg) { dbgprintf("GnuTLS log msg, level %d: %s\n", level, msg); } +#endif #endif /* #ifdef DEBUG */ diff --git a/tools/msggen.c b/tools/msggen.c index 7990a3c8..06244c18 100644 --- a/tools/msggen.c +++ b/tools/msggen.c @@ -24,7 +24,7 @@ #include #include -int main(int argc, char *argv[]) +int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) { int i; -- cgit From 1fb5cee04dfd4d40e64e26a0c622640781cd06f7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 17:53:33 +0200 Subject: improved testbench - added tests for queue disk-only mode --- ChangeLog | 2 ++ tests/Makefile.am | 4 +++- tests/diskqueue.sh | 31 +++++++++++++++++++++++++++++++ tests/testsuites/diskqueue.conf | 16 ++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 tests/diskqueue.sh create mode 100644 tests/testsuites/diskqueue.conf diff --git a/ChangeLog b/ChangeLog index 48343b5f..85411722 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- improved testbench + * added tests for queue disk-only mode (checks disk queue logic) --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/tests/Makefile.am b/tests/Makefile.am index 87dca985..0f4cbce1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh diskqueue.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -27,6 +27,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ parsertest.sh \ + diskqueue.sh \ + testsuites/diskqueue.conf \ manytcp.sh \ testsuites/manytcp.conf \ omod-if-array.sh \ diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh new file mode 100755 index 00000000..6384eb64 --- /dev/null +++ b/tests/diskqueue.sh @@ -0,0 +1,31 @@ +# Test for disk-only queue mode +# This test checks if queue files can be correctly written +# and read back, but it does not test the transition from +# memory to disk mode for DA queues. +# added 2009-04-17 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo testing queue disk-only mode +rm -rf test-spool +mkdir test-spool +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# 20000 messages should be enough - the disk test is slow enough ;) +./tcpflood 127.0.0.1 13514 1 20000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 4 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 19999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +rm -rf test-spool diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf new file mode 100644 index 00000000..8851a459 --- /dev/null +++ b/tests/testsuites/diskqueue.conf @@ -0,0 +1,16 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit From aa43d7f83125f20f8efdf4152bdd9a09e7a81495 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Apr 2009 16:32:33 +0200 Subject: doc: added (hopefully) easier to grasp queue explanation --- ChangeLog | 2 + doc/Makefile.am | 16 ++- doc/dataflow.png | Bin 0 -> 24601 bytes doc/direct_queue0.png | Bin 0 -> 2048 bytes doc/direct_queue1.png | Bin 0 -> 2979 bytes doc/direct_queue2.png | Bin 0 -> 4117 bytes doc/direct_queue3.png | Bin 0 -> 4430 bytes doc/direct_queue_directq.png | Bin 0 -> 10075 bytes doc/direct_queue_rsyslog.png | Bin 0 -> 10465 bytes doc/direct_queue_rsyslog2.png | Bin 0 -> 12350 bytes doc/queue_analogy_tv.png | Bin 0 -> 18730 bytes doc/queues.html | 14 ++- doc/queues_analogy.html | 259 ++++++++++++++++++++++++++++++++++++++ doc/src/dataflow.dia | Bin 0 -> 2662 bytes doc/src/direct_queue0.dia | Bin 0 -> 966 bytes doc/src/direct_queue1.dia | Bin 0 -> 1058 bytes doc/src/direct_queue2.dia | Bin 0 -> 1143 bytes doc/src/direct_queue3.dia | Bin 0 -> 1214 bytes doc/src/direct_queue_directq.dia | Bin 0 -> 1705 bytes doc/src/direct_queue_rsyslog.dia | Bin 0 -> 2311 bytes doc/src/direct_queue_rsyslog2.dia | Bin 0 -> 2638 bytes doc/src/queue_analogy_tv.dia | Bin 0 -> 1749 bytes 22 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 doc/dataflow.png create mode 100644 doc/direct_queue0.png create mode 100644 doc/direct_queue1.png create mode 100644 doc/direct_queue2.png create mode 100644 doc/direct_queue3.png create mode 100644 doc/direct_queue_directq.png create mode 100644 doc/direct_queue_rsyslog.png create mode 100644 doc/direct_queue_rsyslog2.png create mode 100644 doc/queue_analogy_tv.png create mode 100644 doc/queues_analogy.html create mode 100644 doc/src/dataflow.dia create mode 100644 doc/src/direct_queue0.dia create mode 100644 doc/src/direct_queue1.dia create mode 100644 doc/src/direct_queue2.dia create mode 100644 doc/src/direct_queue3.dia create mode 100644 doc/src/direct_queue_directq.dia create mode 100644 doc/src/direct_queue_rsyslog.dia create mode 100644 doc/src/direct_queue_rsyslog2.dia create mode 100644 doc/src/queue_analogy_tv.dia diff --git a/ChangeLog b/ChangeLog index 85411722..976d8084 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- improved doc + * added (hopefully) easier to grasp queue explanation - improved testbench * added tests for queue disk-only mode (checks disk queue logic) --------------------------------------------------------------------------- diff --git a/doc/Makefile.am b/doc/Makefile.am index 0f3dca70..4d9d94ff 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -109,6 +109,20 @@ html_files = \ rsyslog_conf_output.html \ rsyslog_conf_templates.html \ rsyslog_conf_nomatch.html \ + queues_analogy.html \ src/classes.dia -EXTRA_DIST = $(html_files) +grfx_files = \ + direct_queue0.png \ + direct_queue1.png \ + direct_queue2.png \ + direct_queue3.png \ + direct_queue_rsyslog.png \ + direct_queue_rsyslog2.png \ + direct_queue_directq.png \ + dataflow.png \ + queue_analogy_tv.png \ + gssapi.png \ + rsyslog-vers.png + +EXTRA_DIST = $(html_files) $(grfx_files) diff --git a/doc/dataflow.png b/doc/dataflow.png new file mode 100644 index 00000000..fd614d8c Binary files /dev/null and b/doc/dataflow.png differ diff --git a/doc/direct_queue0.png b/doc/direct_queue0.png new file mode 100644 index 00000000..6d1b373f Binary files /dev/null and b/doc/direct_queue0.png differ diff --git a/doc/direct_queue1.png b/doc/direct_queue1.png new file mode 100644 index 00000000..503f8151 Binary files /dev/null and b/doc/direct_queue1.png differ diff --git a/doc/direct_queue2.png b/doc/direct_queue2.png new file mode 100644 index 00000000..cbb99af8 Binary files /dev/null and b/doc/direct_queue2.png differ diff --git a/doc/direct_queue3.png b/doc/direct_queue3.png new file mode 100644 index 00000000..cc49299f Binary files /dev/null and b/doc/direct_queue3.png differ diff --git a/doc/direct_queue_directq.png b/doc/direct_queue_directq.png new file mode 100644 index 00000000..c5d8769d Binary files /dev/null and b/doc/direct_queue_directq.png differ diff --git a/doc/direct_queue_rsyslog.png b/doc/direct_queue_rsyslog.png new file mode 100644 index 00000000..6150222d Binary files /dev/null and b/doc/direct_queue_rsyslog.png differ diff --git a/doc/direct_queue_rsyslog2.png b/doc/direct_queue_rsyslog2.png new file mode 100644 index 00000000..807b064d Binary files /dev/null and b/doc/direct_queue_rsyslog2.png differ diff --git a/doc/queue_analogy_tv.png b/doc/queue_analogy_tv.png new file mode 100644 index 00000000..fedcb558 Binary files /dev/null and b/doc/queue_analogy_tv.png differ diff --git a/doc/queues.html b/doc/queues.html index 41c5865f..4a9509a0 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -1,6 +1,5 @@ - Understanding rsyslog queues back @@ -10,6 +9,13 @@ queue, one part of the system "produces" something while another part "consumes" this something. The "something" is most often syslog messages, but queues may also be used for other purposes.

    +

    This document provides a good insight into technical details, operation modes +and implications. In addition to it, an +rsyslog queue concepts overview document +exists which tries to explain queues with the help of some analogies. This may +probably be a better place to start reading about queues. I assume that once you +have understood that document, the material here will be much easier to grasp +and look much more natural.

    The most prominent example is the main message queue. Whenever rsyslog receives a message (e.g. locally, via UDP, TCP or in whatever else way), it places these messages into the main message queue. Later, it is dequeued by the @@ -18,7 +24,7 @@ front of each action, there is also a queue, which potentially de-couples the filter processing from the actual action (e.g. writing to file, database or forwarding to another host).

    Where are Queues Used?

    -

     Currently, queues are used for the main message queue and for the +

    Currently, queues are used for the main message queue and for the actions.

    There is a single main message queue inside rsyslog. Each input module delivers messages to it. The main message queue worker filters messages based on @@ -354,8 +360,8 @@ save.

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html new file mode 100644 index 00000000..1584c66d --- /dev/null +++ b/doc/queues_analogy.html @@ -0,0 +1,259 @@ + + +turning lanes and rsyslog queues - an analogy + +back + +

    Turning Lanes and Rsyslog Queues - an Analogy

    +

    If there is a single object absolutely vital to understanding the way +rsyslog works, this object is queues. Queues offer a variety of services, +including support for multithreading. While there is elaborate in-depth +documentation on the ins and outs of rsyslog queues, +some of the concepts are hard to grasp even for experienced people. I think this +is because rsyslog uses a very high layer of abstraction which includes things +that look quite unnatural, like queues that do not actually queue... +

    With this document, I take a different approach: I will not describe every specific +detail of queue operation but hope to be able to provide the core idea of how +queues are used in rsyslog by using an analogy. I will compare the rsyslog data flow +with real-life traffic flowing at an intersection. +

    But first let's set the stage for the rsyslog part. The graphic below describes +the data flow inside rsyslog: +

    rsyslog data flow +

    Note that there is a video tutorial +available on the data flow. It is not perfect, but may aid in understanding this picture. +

    For our needs, the important fact to know is that messages enter rsyslog on "the +left side" (for example, via UDP), are being preprocessed, put into the +so-called main queue, taken off that queue, be filtered and be placed into one or +several action queues (depending on filter results). They leave rsyslog on "the +right side" where output modules (like the file or database writer) consume them. +

    So there are always two stages where a message (conceptually) is queued - first +in the main queue and later on in n action specific queues (with n being the number of +actions that the message in question needs to be processed by, what is being decided +by the "Filter Engine"). As such, a message will be in at least two queues +during its lifetime (with the exeception of messages being discarded by the queue itself, +but for the purpose of this document, we will ignore that possibility). +

    Also, it is vitally +important to understand that each action has a queue sitting in front of it. +If you have dug into the details of rsyslog configuration, you have probably seen that +a queue mode can be set for each action. And the default queue mode is the so-called +"direct mode", in which "the queue does not actually enqueue data". +That sounds silly, but is not. It is an important abstraction that helps keep the code clean. +

    To understand this, we first need to look at who is the active component. In our data flow, +the active part always sits to the left of the object. For example, the "Preprocessor" +is being called by the inputs and calls itself into the main message queue. That is, the queue +receiver is called, it is passive. One might think that the "Parser & Filter Engine" +is an active component that actively pulls messages from the queue. This is wrong! Actually, +it is the queue that has a pool of worker threads, and these workers pull data from the queue +and then call the passively waiting Parser and Filter Engine with those messages. So the +main message queue is the active part, the Parser and Filter Engine is passive. +

    Let's now try an anlogy analogy for this part: Think about a TV show. The show is produced +in some TV studio, from there sent (actively) to a radio tower. The radio tower passively +receives from the studio and then actively sends out a signal, which is passively received +by your TV set. In our simplified view, we have the following picture: +

    rsyslog queues and TV analogy +

    The lower part of the picture lists the equivalent rsyslog entities, in an abstracted way. +Every queue has a producer (in the above sample the input) and a consumer (in the above sample the Parser +and Filter Engine). Their active and passive functions are equivalent to the TV entities +that are listed on top of the rsyslog entity. For example, a rsyslog consumer can never +actively initate reception of a message in the same way a TV set can not actively +"initiate" a TV show - both can only "handle" (display or process) +what is sent to them. +

    Now let's look at the action queues: here, the active part, the producer, is the +Parser and Filter Engine. The passive part is the Action Processor. The later does any +processing that is necessary to call the output plugin, in particular it processes the template +to create the plugin calling parameters (eiter a string or vector of arguments). From the +action queue's point of view, Action Processor and Output form a single entity. Again, the +TV set analogy holds. The Output does not actively ask the queue for data, but +rater passively waits until the queue itself pushes some data to it. + +

    Armed with this knowledge, we can now look at the way action queue modes work. My analogy here +is a junction, as shown below (note that the colors in the pictures below are not related to +the colors in the pictures above!): +

    +

    This is a very simple real-life traffic case: one road joins another. We look at +traffic on the straight road, here shown by blue and green arrows. Traffic in the +opposing direction is shown in blue. Traffic flows without +any delays as long as nobody takes turns. To be more precise, if the opposing traffic takes +a (right) turn, traffic still continues to flow without delay. However, if a car in the red traffic +flow intend to do a (left, then) turn, the situation changes: +

    +

    The turning car is represented by the green arrow. It can not turn unless there is a gap +in the "blue traffic stream". And as this car blocks the roadway, the remaining +traffic (now shown in red, which should indicate the block condition), +must wait until the "green" car has made its turn. So +a queue will build up on that lane, waiting for the turn to be completed. +Note that in the examples below I do not care that much about the properties of the +opposing traffic. That is, because its structure is not really important for what I intend to +show. Think about the blue arrow as being a traffic stream that most of the time blocks +left-turners, but from time to time has a gap that is sufficiently large for a left-turn +to complete. +

    Our road network designers know that this may be unfortunate, and for more important roads +and junctions, they came up with the concept of turning lanes: +

    +

    Now, the car taking the turn can wait in a special area, the turning lane. As such, +the "straight" traffic is no longer blocked and can flow in parallel to the +turning lane (indicated by a now-green-again arrow). + +

    However, the turning lane offers only finite space. So if too many cars intend to +take a left turn, and there is no gap in the "blue" traffic, we end up with +this well-known situation: +

    +

    The turning lane is now filled up, resulting in a tailback of cars intending to +left turn on the main driving lane. The end result is that "straight" traffic +is again being blocked, just as in our initial problem case without the turning lane. +In essence, the turning lane has provided some relief, but only for a limited amount of +cars. Street system designers now try to weight cost vs. benefit and create (costly) +turning lanes that are sufficiently large to prevent traffic jams in most, but not all +cases. +

    Now let's dig a bit into the mathematical properties of turning lanes. We assume that +cars all have the same length. So, units of cars, the length is alsways one (which is nice, +as we don't need to care about that factor any longer ;)). A turning lane has finite capacity of +n cars. As long as the number of cars wanting to take a turn is less than or eqal +to n, "straigth traffic" is not blocked (or the other way round, traffic +is blocked if at least n + 1 cars want to take a turn!). We can now find an optimal +value for n: it is a function of the probability that a car wants to turn +and the cost of the turning lane +(as well as the probability there is a gap in the "blue" traffic, but we ignore this +in our simple sample). +If we start from some finite upper bound of n, we can decrease +n to a point where it reaches zero. But let's first look at n = 1, in which case exactly +one car can wait on the turning lane. More than one car, and the rest of the traffic is blocked. +Our everyday logic indicates that this is actually the lowest boundary for n. +

    In an abstract view, however, n can be zero and that works nicely. There still can be +n cars at any given time on the turning lane, it just happens that this means there can +be no car at all on it. And, as usual, if we have at least n + 1 cars wanting to turn, +the main traffic flow is blocked. True, but n + 1 = 0 + 1 = 1 so as soon as there is any +car wanting to take a turn, the main traffic flow is blocked (remeber, in all cases, I assume +no sufficently large gaps in the opposing trafic). +

    This is the situation our everyday perception calls "road without turning lane". +In my math model, it is a "road with turning lane of size 0". The subtle difference +is important: my math model guarantees that, in an abstract sense, there always is a turning +lane, it may just be too short. But it exists, even though we don't see it. And now I can +claim that even in my small home village, all roads have turning lanes, which is rather +impressive, isn't it? ;) +

    And now we finally have arrived at rsyslog's queues! Rsyslog action queues exists for +all actions just like all roads in my village have turning lanes! And as in this real-life sample, +it may be hard to see the action queues for that reason. In rsyslog, the "direct" queue +mode is the equivalent to the 0-sized turning lane. And actions queues are the equivalent to turning +lanes in general, with our real-life n being the maximum queue size. +The main traffic line (which sometimes is blocked) is the equivalent to the +main message queue. And the periods without gaps in the opposing traffic are equivalent to +execution time of an action. In a rough sketch, the rsyslog main and action queues look like in the +following picture. +

    +

    We need to read this picture from right to left (otherwise I would need to redo all +the graphics ;)). In action 3, you see a 0-sized turning lane, aka an action queue in "direct" +mode. All other queues are run in non-direct modes, but with different sizes greater than 0. +

    Let us first use our car analogy: +Assume we are in a car on the main lane that wants to take turn into the "action 4" +road. We pass action 1, where a number of cars wait in the turning lane and we pass +action 2, which has a slightly smaller, but still not filled up turning lane. So we pass that +without delay, too. Then we come to "action 4", which has no turning lane. Unfortunately, +the car in front of us wants to turn left into that road, so it blocks the main lane. So, this time +we need to wait. An observer standing on the sidewalk may see that while we need to wait, there are +still some cars in the "action 4" turning lane. As such, even though no new cars can +arrive on the main lane, cars still turn into the "action 4" lane. In other words, +an observer standing in "action 4" road is unable to see that traffic on the main lane +is blocked. +

    Now on to rsyslog: Other than in the real-world traffic example, messages in rsyslog +can - at more or less the +same time - "take turns" into several roads at once. This is done by duplicating the message +if the road has a non-zero-sized "turning lane" - or in rsyslog terms a queue that is +running in any non-direct mode. If so, a deep copy of the message object is made, that placed into +the action queue and then the initial message proceeds on the "main lane". The action +queue then pushes the duplicates through action processing. This is also the reason why a +discard action inside a non-direct queue does not seem to have an effect. Actually, it discards the +copy that was just created, but the original message object continues to flow. +

    +In action 1, we have some entries in the action queue, as we have in action 2 (where the queue is +slightly shorter). As we have seen, new messages pass action one and two almost instantaneously. +However, when a messages reaches action 3, its flow is blocked. Now, message processing must wait +for the action to complete. Processing flow in a direct mode queue is something like a U-turn: + +

    message processing in an rsyslog action queue in direct mode +

    The message starts to execute the action and once this is done, processing flow continues. +In a real-life analogy, this may be the route of a delivery man who needs to drop a parcel +in a side street before he continues driving on the main route. As a side-note, think of what happens +with the rest of the delivery route, at least for today, if the delivery truck has a serious accident +in the side street. The rest of the parcels won't be delivered today, will they? This is exactly how the +discard action works. It drops the message object inside the action and thus the message will no +longer be available for further delivery - but as I said, only if the discard is done in a +direct mode queue (I am stressing this example because it often causes a lot of confusion). +

    Back to the overall scenario. We have seen that messages need to wait for action 3 to +complete. Does this necessarily mean that at the same time no messages can be processed +in action 4? Well, it depends. As in the real-life scenario, action 4 will continue to +receive traffic as long as its action queue ("turn lane") is not drained. In +our drawing, it is not. So action 4 will be executed while messages still wait for action 3 +to be completed. +

    Now look at the overall picture from a slightly different angle: +

    message processing in an rsyslog action queue in direct mode +

    The number of all connected green and red arrows is four - one each for action 1, 2 and 3 +(this one is dotted as action 4 was a special case) and one for the "main lane" as +well as acton 3 (this one contains the sole red arrow). This number is the lower bound for +the number of threads in rsyslog's output system ("right-hand part" of the main message +queue)! Each of the connected arrows is a continous thread and each "turn lane" is +a place where processing is forked onto a new thread. Also, note that in action 3 the processing +is carried out on the main thread, but not in the non-direct queue modes. +

    I have said this is "the lower bound for the number of threads...". This is with +good reason: the main queue may have more than one worker thread (individual action queues +currently do not support this, but could do in the future - there are good reasons for that, too +but exploring why would finally take us away from what we intend to see). Note that you +configure an upper bound for the number of main message queue worker threads. The actual number +varies depending on a lot of operational variables, most importantly the number of messages +inside the queue. The number t_m of actually running threads is within the integer-interval +[0,confLimit] (with confLimit being the operator configured limit, which defaults to 5). +Output plugins may have more than one thread created by themselves. It is quite unusual for an +output plugin to create such threads and so I assume we do not have any of these. +Then, the overall number of threads in rsyslog's filtering and output system is +t_total = t_m + number of actions in non-direct modes. Add the number of +inputs configured to that and you have the total number of threads running in rsyslog at +a given time (assuming again that inputs utilize only one thread per plugin, a not-so-safe +assumption). +

    A quick side-note: I gave the lower bound for t_m as zero, which is somewhat in contrast +to what I wrote at the begin of the last paragraph. Zero is actually correct, because rsyslog +stops all worker threads when there is no work to do. This is also true for the action queues. +So the ultimate lower bound for a rsyslog output system without any work to carry out actually is zero. +But this bound will never be reached when there is continous flow of activity. And, if you are +curios: if the number of workers is zero, the worker wakeup process is actually handled within the +threading context of the "left-hand-side" (or producer) of the queue. After being +started, the worker begins to play the active queue component again. All of this, of course, +can be overridden with configuraton directives. +

    When looking at the threading model, one can simply add n lanes to the main lane but otherwise +retain the traffic analogy. This is a very good description of the actual process (think what this +means to the "turning lanes"; hint: there still is only one per action!). +

    Let's try to do a warp-up: I have hopefully been able to show that in rsyslog, an action +queue "sits in front of" each output plugin. Messages are received and flow, from input +to output, over various stages and two level of queues to the outputs. Actions queues are always +present, but may not easily be visible when in direct mode (where no actual queueing takes place). +The "road junktion with turning lane" analogy well describes the way - and intent - of the various +queue levels in rsyslog. +

    On the output side, the queue is the active component, not the consumer. As such, the consumer +can not ask the queue for anything (like n number of messages) but rather is activated by the queue +itself. As such, a queue somewhat resembles a "living thing" whereas the outputs are +just tools that this "living thing" uses. +

    Note that I left out a couple of subtleties, especially when it comes +to error handling and terminating +a queue (you hopefully have now at least a rough idea why I say "terminating a queue" +and not "terminating an action" - who is the "living thing"?). An action returns +a status to the queue, but it is the queue that ultimately decides which messages can finally be +considered processed and which not. Please note that the queue may even cancel an output right in +the middle of its action. This happens, if configured, if an output needs more than a configured +maximum processing time and is a guard condition to prevent slow outputs from defering a rsyslog +restart for too long. Especially in this case re-queueing and cleanup is not trivial. Also, note that +I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additonal +constraints, especially in regard to the threading model. Transitioning between actual +disk-assisted mode and pure-in-memory-mode (which is done automatically when needed) is also far from +trivial and a real joy for an implementor to work on ;). +

    If you have not done so before, it may be worth reading the +rsyslog queue user's guide, which most importantly lists all +the knobs you can turn to tweak queue operation. +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 3 or higher.

    + + diff --git a/doc/src/dataflow.dia b/doc/src/dataflow.dia new file mode 100644 index 00000000..3875fc61 Binary files /dev/null and b/doc/src/dataflow.dia differ diff --git a/doc/src/direct_queue0.dia b/doc/src/direct_queue0.dia new file mode 100644 index 00000000..4446619b Binary files /dev/null and b/doc/src/direct_queue0.dia differ diff --git a/doc/src/direct_queue1.dia b/doc/src/direct_queue1.dia new file mode 100644 index 00000000..7a64ea09 Binary files /dev/null and b/doc/src/direct_queue1.dia differ diff --git a/doc/src/direct_queue2.dia b/doc/src/direct_queue2.dia new file mode 100644 index 00000000..b0c394c0 Binary files /dev/null and b/doc/src/direct_queue2.dia differ diff --git a/doc/src/direct_queue3.dia b/doc/src/direct_queue3.dia new file mode 100644 index 00000000..bc477b25 Binary files /dev/null and b/doc/src/direct_queue3.dia differ diff --git a/doc/src/direct_queue_directq.dia b/doc/src/direct_queue_directq.dia new file mode 100644 index 00000000..37fdb44c Binary files /dev/null and b/doc/src/direct_queue_directq.dia differ diff --git a/doc/src/direct_queue_rsyslog.dia b/doc/src/direct_queue_rsyslog.dia new file mode 100644 index 00000000..9a030117 Binary files /dev/null and b/doc/src/direct_queue_rsyslog.dia differ diff --git a/doc/src/direct_queue_rsyslog2.dia b/doc/src/direct_queue_rsyslog2.dia new file mode 100644 index 00000000..c596f39f Binary files /dev/null and b/doc/src/direct_queue_rsyslog2.dia differ diff --git a/doc/src/queue_analogy_tv.dia b/doc/src/queue_analogy_tv.dia new file mode 100644 index 00000000..00fbdeb5 Binary files /dev/null and b/doc/src/queue_analogy_tv.dia differ -- cgit