From 51e690f720b4a53a431e7a536b2fe8c25e866f7d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Mar 2009 18:54:42 +0100 Subject: fixed some portability issues first noticed on FreeBSD Also, some cosmetic improvements. --- plugins/omstdout/omstdout.c | 2 +- tests/getline.c | 3 ++- tests/nettester.c | 4 +++- tests/omod-if-array.sh | 9 ++++++++- tests/parsertest.sh | 8 +++++++- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 7c63b5c4..181895a4 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -110,7 +110,7 @@ CODESTARTdoAction if(iParam > 0) szBuf[iBuf++] = ','; /* all but first need a delimiter */ iParamVal = 0; - while(szParams[iParam][iParamVal] != '\0' && iBuf < sizeof(szBuf)) { + while(szParams[iParam][iParamVal] != '\0' && iBuf < (int) sizeof(szBuf)) { szBuf[iBuf++] = szParams[iParam][iParamVal++]; } ++iParam; diff --git a/tests/getline.c b/tests/getline.c index 10de2ffe..617d1b0e 100644 --- a/tests/getline.c +++ b/tests/getline.c @@ -23,7 +23,8 @@ */ #include "config.h" #include -#include +#include +#include /* we emulate getline (the dirty way) if we do not have it * We do not try very hard, as this is just a test driver. diff --git a/tests/nettester.c b/tests/nettester.c index 89a784f3..37183ac9 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #define EXIT_FAILURE 1 @@ -361,7 +362,8 @@ int main(int argc, char *argv[]) if((srcdir = getenv("srcdir")) == NULL) srcdir = "."; - printf("Start of nettester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s)\n", + srcdir, testSuite, argv[2]); /* create input config file */ if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index 8a8d67f3..fd845b4d 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,5 +1,12 @@ -#!/bin/bash -e echo test omod-if-array via udp ./nettester omod-if-array udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + echo test omod-if-array via tcp ./nettester omod-if-array tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi + diff --git a/tests/parsertest.sh b/tests/parsertest.sh index fabe7e8d..a6b7d45c 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,5 +1,11 @@ -#!/bin/bash -e echo test parsertest via udp ./nettester parse1 udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + echo test parsertest via tcp ./nettester parse1 tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi -- cgit From 97480eafbc67ec7e84497868a1777ce0d7881e6c Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:24 +0100 Subject: Start the output module for Oracle. Currently, resources are allocated, freed and the code compiles. No tests yet. --- plugins/omoracle/Makefile.am | 8 +++ plugins/omoracle/omoracle.c | 139 +++++++++++++++++++++++++++++++++++++++++++ plugins/omoracle/omoracle.h | 23 +++++++ 3 files changed, 170 insertions(+) create mode 100644 plugins/omoracle/Makefile.am create mode 100644 plugins/omoracle/omoracle.c create mode 100644 plugins/omoracle/omoracle.h diff --git a/plugins/omoracle/Makefile.am b/plugins/omoracle/Makefile.am new file mode 100644 index 00000000..6b75218f --- /dev/null +++ b/plugins/omoracle/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omoracle.la + +omoracle_la_SOURCES = omoracle.c omoracle.h +omoracle_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(ORACLE_CFLAGS) +omoracle_la_LDFLAGS = -module -avoid-version +omoracle_la_LIBADD = $(ORACLE_LIBS) + +EXTRA_DIST = diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c new file mode 100644 index 00000000..696cd908 --- /dev/null +++ b/plugins/omoracle/omoracle.c @@ -0,0 +1,139 @@ +/** omoracle.c + + This is an output module feeding directly to an Oracle + database. It uses Oracle Call Interface, a propietary module + provided by Oracle. + + Author: Luis Fernando Muñoz Mejías + + + This file is part of rsyslog. +*/ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" +#include "omoracle.h" + +MODULE_TYPE_OUTPUT + +/** */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +typedef struct _instanceData { + OCIEnv* environment; + OCISession* session; + OCIError* error; + OCIServer* server; + OCIStmt* statement; + OCISvcCtx* service; + OCIAuthInfo* authinfo; + OCIBind* binding; +} instanceData; + +/** Generic function for handling errors from OCI. + + It will be called only inside CHECKERR and CHECKENV macros. + + Arguments: handle The error or environment handle to check. + htype: OCI_HTYPE_* constant, usually OCI_HTYPE_ERROR or + OCI_HTYPE_ENV + status: status code to check, usually the return value of an OCI + function. + + Returns OCI_SUCCESS on success, OCI_ERROR otherwise. +*/ +static int oci_errors(void* handle, ub4 htype, sword status) +{ + sb4 errcode; + char buf[MAX_BUFSIZE]; + + switch (status) { + case OCI_SUCCESS: + return OCI_SUCCESS; + break; + case OCI_SUCCESS_WITH_INFO: + printf ("OCI SUCCESS - With info\n"); + break; + case OCI_NEED_DATA: + printf ("OCI NEEDS MORE DATA\n"); + break; + case OCI_ERROR: + printf ("OCI GENERAL ERROR\n"); + if (handle) { + OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); + printf ("Error message: %s", buf); + } else + printf ("NULL handle\n" + "Unable to extract further information"); + break; + case OCI_INVALID_HANDLE: + printf ("OCI INVALID HANDLE\n"); + break; + case OCI_STILL_EXECUTING: + printf ("Still executing...\n"); + break; + case OCI_CONTINUE: + printf ("OCI CONTINUE\n"); + break; + } + return OCI_ERROR; +} + + +/* Resource allocation */ +BEGINcreateInstance +CODESTARTcreateInstance +CHECKENV(pData->environment, + OCIEnvCreate(&(pData->environment), OCI_DEFAULT, + NULL, NULL, NULL, NULL, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->error), + OCI_HTYPE_ERROR, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->server), + OCI_HTYPE_SERVER, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->service), + OCI_HTYPE_SVCCTX, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->authinfo), + OCI_HTYPE_AUTHINFO, 0, NULL)); +finalize_it: +ENDcreateInstance + +/** Free any resources allocated by createInstance. */ +BEGINfreeInstance +CODESTARTfreeInstance + +OCIHandleFree(pData->environment, OCI_HTYPE_ENV); +OCIHandleFree(pData->error, OCI_HTYPE_ERROR); +OCIHandleFree(pData->server, OCI_HTYPE_SERVER); +OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); +OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); + +RETiRet; + +ENDfreeInstance + +/* BEGINmodInit() */ +/* CODESTARTmodInit */ +/* *ipIFVersProvided = CURR_MOD_IF_VERSION; */ +/* CODEmodInit_QueryRegCFSLineHdlr */ +/* CHKiRet(objUse(errmsg, CORE_COMPONENT)); */ +/* ENDmodInit */ diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h new file mode 100644 index 00000000..b0e70917 --- /dev/null +++ b/plugins/omoracle/omoracle.h @@ -0,0 +1,23 @@ +/** Definitions for the Oracle output module. + + This module needs OCI to be installed (on Red Hat-like systems + this is usually the oracle-instantclient-devel RPM). + + Author: Luis Fernando Muñoz Mejías +*/ +#ifndef __OMORACLEH__ +#define __OMORACLEH__ + +/** Macros to make error handling easier. */ + +/** Checks for errors on the OCI handling. */ +#define CHECKERR(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ERROR, (status))) + +/** Checks for errors when handling the environment of OCI. */ +#define CHECKENV(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ENV, (status))) + +enum { MAX_BUFSIZE = 2048 }; + +#endif -- cgit From b8f10ad00853275045180738b91e90a80c7693cc Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:25 +0100 Subject: Include omoracle in the build system. Add configure option to build the oracle support, named --enable-oracle and fix the Makefile.am accordingly. --- Makefile.am | 7 ++++++- configure.ac | 36 +++++++++++++++++++++++++++++++++++- plugins/omoracle/Makefile.am | 2 +- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 87e378ee..a5c3e267 100644 --- a/Makefile.am +++ b/Makefile.am @@ -110,15 +110,20 @@ if ENABLE_RFC3195 SUBDIRS += plugins/im3195 endif +if ENABLE_ORACLE +SUBDIRS += plugins/omoracle +endif + # tests are added as last element, because tests may need different # modules that need to be generated first SUBDIRS += tests + # make sure "make distcheck" tries to build all modules. This means that # a developer must always have an environment where every supporting library # is available. If that is not the case, the respective configure option may # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-oracle ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 0c9483ca..9be3c7ca 100644 --- a/configure.ac +++ b/configure.ac @@ -452,7 +452,39 @@ AM_CONDITIONAL(ENABLE_PGSQL, test x$enable_pgsql = xyes) AC_SUBST(PGSQL_CFLAGS) AC_SUBST(PGSQL_LIBS) - +# oracle (OCI) support +AC_ARG_ENABLE(oracle, + [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_oracle="yes" ;; + no) enable_oracle="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-oracle) ;; + esac], + [enable_oracle=no] +) +if test "x$enable_oracle" = "xyes"; then + AC_CHECK_PROG( + [HAVE_ORACLE_CONFIG], + [oracle-instantclient-config], + [yes],,, + ) + if test "x${HAVE_ORACLE_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([oracle-instantclient-config not found in PATH]) + fi + AC_CHECK_LIB( + [occi], + [OCIEnvCreate], + [ORACLE_CFLAGS="`oracle-instantclient-config --cflags`" + ORACLE_LIBS="`oracle-instantclient-config --libs`" + ], + [AC_MSG_FAILURE([Oracle (OCI) libraray is missing])], + [`oracle-instantclient-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ORACLE, test x$enable_oracle = xyes) +AC_SUBST(ORACLE_CFLAGS) +AC_SUBST(ORACLE_LIBS) + # libdbi support AC_ARG_ENABLE(libdbi, [AS_HELP_STRING([--enable-libdbi],[Enable libdbi database support @<:@default=no@:>@])], @@ -712,6 +744,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omlibdbi/Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ + plugins/omoracle/Makefile \ tests/Makefile]) AC_OUTPUT @@ -725,6 +758,7 @@ echo "Zlib compression support enabled: $enable_zlib" echo "MySql support enabled: $enable_mysql" echo "libdbi support enabled: $enable_libdbi" echo "PostgreSQL support enabled: $enable_pgsql" +echo "Oracle (OCI) support enabled: $enable_oracle" echo "SNMP support enabled: $enable_snmp" echo "Mail support enabled: $enable_mail" echo "RELP support enabled: $enable_relp" diff --git a/plugins/omoracle/Makefile.am b/plugins/omoracle/Makefile.am index 6b75218f..11257fb2 100644 --- a/plugins/omoracle/Makefile.am +++ b/plugins/omoracle/Makefile.am @@ -5,4 +5,4 @@ omoracle_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(ORACLE_CFLAGS) omoracle_la_LDFLAGS = -module -avoid-version omoracle_la_LIBADD = $(ORACLE_LIBS) -EXTRA_DIST = +#EXTRA_DIST = -- cgit From 57dcc5c6c96aa72b24c5f8c9952f789d08943383 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:26 +0100 Subject: Add all other blocks (macros) needed to make this module work. At this stage they are all empty, but at least it should be possible to instantiate the module and perform some basic tests. Fix some compilation warnings --- plugins/omoracle/omoracle.c | 61 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 696cd908..6077594f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -61,7 +61,7 @@ typedef struct _instanceData { static int oci_errors(void* handle, ub4 htype, sword status) { sb4 errcode; - char buf[MAX_BUFSIZE]; + unsigned char buf[MAX_BUFSIZE]; switch (status) { case OCI_SUCCESS: @@ -76,7 +76,8 @@ static int oci_errors(void* handle, ub4 htype, sword status) case OCI_ERROR: printf ("OCI GENERAL ERROR\n"); if (handle) { - OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); + OCIErrorGet(handle, 1, NULL, &errcode, buf, + sizeof buf, htype); printf ("Error message: %s", buf); } else printf ("NULL handle\n" @@ -100,19 +101,19 @@ static int oci_errors(void* handle, ub4 htype, sword status) BEGINcreateInstance CODESTARTcreateInstance CHECKENV(pData->environment, - OCIEnvCreate(&(pData->environment), OCI_DEFAULT, + OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->error), + OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->server), + OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->service), + OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->authinfo), + OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); finalize_it: ENDcreateInstance @@ -131,9 +132,43 @@ RETiRet; ENDfreeInstance -/* BEGINmodInit() */ -/* CODESTARTmodInit */ -/* *ipIFVersProvided = CURR_MOD_IF_VERSION; */ -/* CODEmodInit_QueryRegCFSLineHdlr */ -/* CHKiRet(objUse(errmsg, CORE_COMPONENT)); */ -/* ENDmodInit */ + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +/* Right now, this module is compatible with nothing. */ +ENDisCompatibleWithFeature + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINdoAction +CODESTARTdoAction +ENDdoAction + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +ENDmodInit -- cgit From 0676277119bd39562a0b8c0de6d2fee23e1deb11 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:27 +0100 Subject: Add handlers on modInit. This avoids crashes on initialization. --- plugins/omoracle/omoracle.c | 78 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 6077594f..517b7173 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -68,29 +68,29 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_SUCCESS; break; case OCI_SUCCESS_WITH_INFO: - printf ("OCI SUCCESS - With info\n"); + dbgprintf ("OCI SUCCESS - With info\n"); break; case OCI_NEED_DATA: - printf ("OCI NEEDS MORE DATA\n"); + dbgprintf ("OCI NEEDS MORE DATA\n"); break; case OCI_ERROR: - printf ("OCI GENERAL ERROR\n"); + dbgprintf ("OCI GENERAL ERROR\n"); if (handle) { OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); - printf ("Error message: %s", buf); + dbgprintf ("Error message: %s", buf); } else - printf ("NULL handle\n" + dbgprintf ("NULL handle\n" "Unable to extract further information"); break; case OCI_INVALID_HANDLE: - printf ("OCI INVALID HANDLE\n"); + dbgprintf ("OCI INVALID HANDLE\n"); break; case OCI_STILL_EXECUTING: - printf ("Still executing...\n"); + dbgprintf ("Still executing...\n"); break; case OCI_CONTINUE: - printf ("OCI CONTINUE\n"); + dbgprintf ("OCI CONTINUE\n"); break; } return OCI_ERROR; @@ -100,21 +100,30 @@ static int oci_errors(void* handle, ub4 htype, sword status) /* Resource allocation */ BEGINcreateInstance CODESTARTcreateInstance + +ASSERT(pData != NULL); + +dbgprintf ("***** OMORACLE ***** Creating instance\n"); CHECKENV(pData->environment, OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created environment\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created error\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created server\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created service\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created authinfo\n"); finalize_it: ENDcreateInstance @@ -122,11 +131,18 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance +dbgprintf ("***** OMORACLE ***** Destroying instance\n"); + OCIHandleFree(pData->environment, OCI_HTYPE_ENV); +dbgprintf ("***** OMORACLE ***** Destroyed environment\n"); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); +dbgprintf ("***** OMORACLE ***** Destroyed error\n"); OCIHandleFree(pData->server, OCI_HTYPE_SERVER); +dbgprintf ("***** OMORACLE ***** Destroyed server\n"); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); +dbgprintf ("***** OMORACLE ***** Destroyed service\n"); OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); +dbgprintf ("***** OMORACLE ***** Destroyed authinfo\n"); RETiRet; @@ -135,40 +151,84 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + +dbgprintf ("***** OMORACLE ***** At tryResume\n"); ENDtryResume BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ +dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); +iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); + +if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); +} + +CHKiRet(createInstance(&pData)); + +p += sizeof ":omoracle:" - 1; +if (*p != ';') { + dbgprintf ("***** OMORACLE ***** Wrong char: %c\n", *p); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); +} +p++; + +CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, + OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + +dbgprintf ("***** OMORACLE ***** Salido\n"); + + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction CODESTARTdoAction +dbgprintf ("***** OMORACLE ***** At doAction\n"); ENDdoAction BEGINmodExit CODESTARTmodExit +dbgprintf ("***** OMORACLE ***** At modExit\n"); ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo +dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); + ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); + ENDqueryEtryPt +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, + void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr - CHKiRet(objUse(errmsg, CORE_COMPONENT)); +CHKiRet(objUse(errmsg, CORE_COMPONENT)); +/* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ +CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, + eCmdHdlrCustomHandler, resetConfigVariables, + NULL, STD_LOADABLE_MODULE_ID)); + +dbgprintf ("***** OMORACLE ***** At modInit\n"); ENDmodInit -- cgit From 3dabb2976a2f259ba1f3bd9823ddd2860edc293d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:28 +0100 Subject: Add the ability to connect to the DB based on the config line. It will read and parse the config line (this code is not yet rock-solid) and connect to the database at initialization time. I also cleaned some debug messages that are not needed anymore. --- plugins/omoracle/omoracle.c | 58 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 517b7173..ab9e7e36 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -103,27 +103,22 @@ CODESTARTcreateInstance ASSERT(pData != NULL); -dbgprintf ("***** OMORACLE ***** Creating instance\n"); CHECKENV(pData->environment, OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created environment\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created error\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created server\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created service\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created authinfo\n"); + finalize_it: ENDcreateInstance @@ -155,6 +150,26 @@ CODESTARTtryResume dbgprintf ("***** OMORACLE ***** At tryResume\n"); ENDtryResume +static rsRetVal startSession(instanceData* pData, char* connection, char* user, + char * password) +{ + DEFiRet; + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, user, + strlen(user), OCI_ATTR_USERNAME, pData->error)); + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, password, + strlen(password), OCI_ATTR_PASSWORD, pData->error)); + CHECKERR(pData->error, + OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, connection, + strlen(connection), NULL, 0, NULL, NULL, NULL, + OCI_DEFAULT)); +finalize_it: + RETiRet; +} + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ @@ -163,6 +178,12 @@ iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct + +char connection_string[MAXHOSTNAMELEN]; +char user[_DB_MAXUNAMELEN]; +char pwd[_DB_MAXPWDLEN]; + + CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -170,20 +191,25 @@ if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); } -CHKiRet(createInstance(&pData)); p += sizeof ":omoracle:" - 1; -if (*p != ';') { - dbgprintf ("***** OMORACLE ***** Wrong char: %c\n", *p); + +if (*p == '\0' || *p == ',') { + dbgprintf ("Wrong char processing module arguments: %c\n", *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } -p++; +CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); +CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); +CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); +p--; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); +CHKiRet(createInstance(&pData)); +CHKiRet(startSession(pData, connection_string, user, pwd)); -dbgprintf ("***** OMORACLE ***** Salido\n"); - +dbgprintf ("omoracle module got all its resources allocated " + "and connected to the DB\n"); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -223,12 +249,16 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; +char dbname[MAX_BUFSIZE]; CODEmodInit_QueryRegCFSLineHdlr +dbgprintf ("***** OMORACLE ***** At modInit\n"); CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); - -dbgprintf ("***** OMORACLE ***** At modInit\n"); +dbgprintf ("***** OMORACLE ***** dbname before = %s\n", dbname); +CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, + NULL, dbname, STD_LOADABLE_MODULE_ID)); +dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); ENDmodInit -- cgit From 47b334bef1ea9e72bc41279c42228724f6e141f6 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:29 +0100 Subject: Add the ability to actually run statements. It now runs SQL statements given as templates. In this case, the template is given on the configuration file and the core passes the SQL statement correctly formatted to doAction. I still need to decide how to structure this for having prepared statements (prepare them at parseSelector time) and then make doAction to only bind arguments and execute. It commits after each statement, which is awfully slow but good enough for the moment. Next step after that is have a buffer of arguments, and make doAction store new data as it arrives, then run the statement only when the buffer is almost full. Or something like that. --- plugins/omoracle/omoracle.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ab9e7e36..c1900a73 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -4,6 +4,13 @@ database. It uses Oracle Call Interface, a propietary module provided by Oracle. + Config lines to be used are of this form: + + :omoracle:dbstring,user,password;StatementTemplate + + All fields are mandatory. The dbstring can be an Oracle easystring + or a DB name, as present in the tnsnames.ora file. + Author: Luis Fernando Muñoz Mejías @@ -39,7 +46,6 @@ typedef struct _instanceData { OCIEnv* environment; OCISession* session; OCIError* error; - OCIServer* server; OCIStmt* statement; OCISvcCtx* service; OCIAuthInfo* authinfo; @@ -109,15 +115,12 @@ CHECKENV(pData->environment, CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->server), - OCI_HTYPE_SERVER, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->service), - OCI_HTYPE_SVCCTX, 0, NULL)); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->statement), + OCI_HTYPE_STMT, 0, NULL)); finalize_it: ENDcreateInstance @@ -126,18 +129,13 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance -dbgprintf ("***** OMORACLE ***** Destroying instance\n"); - +OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->environment, OCI_HTYPE_ENV); -dbgprintf ("***** OMORACLE ***** Destroyed environment\n"); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); -dbgprintf ("***** OMORACLE ***** Destroyed error\n"); -OCIHandleFree(pData->server, OCI_HTYPE_SERVER); -dbgprintf ("***** OMORACLE ***** Destroyed server\n"); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); -dbgprintf ("***** OMORACLE ***** Destroyed service\n"); OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); -dbgprintf ("***** OMORACLE ***** Destroyed authinfo\n"); +OCIHandleFree(pData->statement, OCI_HTYPE_STMT); +dbgprintf ("omoracle freed all its resources\n"); RETiRet; @@ -216,12 +214,21 @@ ENDparseSelectorAct BEGINdoAction CODESTARTdoAction -dbgprintf ("***** OMORACLE ***** At doAction\n"); + dbgprintf("omoracle attempting to execute statement %s\n", *ppString); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, pData->error, *ppString, + strlen(*ppString), OCI_NTV_SYNTAX, + OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, pData->statement, pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); +finalize_it: ENDdoAction BEGINmodExit CODESTARTmodExit -dbgprintf ("***** OMORACLE ***** At modExit\n"); ENDmodExit BEGINdbgPrintInstInfo @@ -262,3 +269,4 @@ CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, NULL, dbname, STD_LOADABLE_MODULE_ID)); dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); ENDmodInit + -- cgit From b6123427cf962e70836c07d1c5c2cf39978673b8 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:30 +0100 Subject: Add proper indentation (despite Emacs) and support for retrying. Emacs doesn't allow for proper indentation with rsyslog's macros (no curly brackets, so it doesn't know where functions start), so I had to manually add such indentation. Add support for retrying actions, namely, disconnect from the DB, re-connecting and re-executing the last prepared statement. Needs to be tested. --- plugins/omoracle/omoracle.c | 168 ++++++++++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 78 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index c1900a73..aa506dca 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -50,6 +50,7 @@ typedef struct _instanceData { OCISvcCtx* service; OCIAuthInfo* authinfo; OCIBind* binding; + char* connection; } instanceData; /** Generic function for handling errors from OCI. @@ -74,29 +75,30 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_SUCCESS; break; case OCI_SUCCESS_WITH_INFO: - dbgprintf ("OCI SUCCESS - With info\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI SUCCESS - With info\n"); break; case OCI_NEED_DATA: - dbgprintf ("OCI NEEDS MORE DATA\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI NEEDS MORE DATA\n"); break; case OCI_ERROR: dbgprintf ("OCI GENERAL ERROR\n"); if (handle) { OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); - dbgprintf ("Error message: %s", buf); + errmsg.LogError(0, NO_ERRCODE, "Error message: %s", buf); } else - dbgprintf ("NULL handle\n" - "Unable to extract further information"); + errmsg.LogError(0, NO_ERRCODE, "NULL handle\n" + "Unable to extract further " + "information"); break; case OCI_INVALID_HANDLE: - dbgprintf ("OCI INVALID HANDLE\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI INVALID HANDLE\n"); break; case OCI_STILL_EXECUTING: - dbgprintf ("Still executing...\n"); + errmsg.LogError(0, NO_ERRCODE, "Still executing...\n"); break; case OCI_CONTINUE: - dbgprintf ("OCI CONTINUE\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI CONTINUE\n"); break; } return OCI_ERROR; @@ -107,45 +109,56 @@ static int oci_errors(void* handle, ub4 htype, sword status) BEGINcreateInstance CODESTARTcreateInstance -ASSERT(pData != NULL); - -CHECKENV(pData->environment, - OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, - NULL, NULL, NULL, NULL, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->error), - OCI_HTYPE_ERROR, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), - OCI_HTYPE_AUTHINFO, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->statement), - OCI_HTYPE_STMT, 0, NULL)); + ASSERT(pData != NULL); + + CHECKENV(pData->environment, + OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, + NULL, NULL, NULL, NULL, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->error), + OCI_HTYPE_ERROR, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), + OCI_HTYPE_AUTHINFO, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->statement), + OCI_HTYPE_STMT, 0, NULL)); finalize_it: ENDcreateInstance -/** Free any resources allocated by createInstance. */ +/** Close the session and free anything allocated by + createInstance. */ BEGINfreeInstance CODESTARTfreeInstance -OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); -OCIHandleFree(pData->environment, OCI_HTYPE_ENV); -OCIHandleFree(pData->error, OCI_HTYPE_ERROR); -OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); -OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); -OCIHandleFree(pData->statement, OCI_HTYPE_STMT); -dbgprintf ("omoracle freed all its resources\n"); - -RETiRet; + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->environment, OCI_HTYPE_ENV); + OCIHandleFree(pData->error, OCI_HTYPE_ERROR); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); + OCIHandleFree(pData->statement, OCI_HTYPE_STMT); + free(pData->connection); + dbgprintf ("omoracle freed all its resources\n"); + RETiRet; ENDfreeInstance - BEGINtryResume CODESTARTtryResume + dbgprintf("Attempting to restart the last action\n"); + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, + pData->connection, + strlen(pData->connection), NULL, 0, + NULL, NULL, NULL, OCI_DEFAULT)); + CHECKERR(pData->error, OCIStmtExecute(pData->service, pData->statement, + pData->error, 1, 0, NULL, NULL, + OCI_DEFAULT)); -dbgprintf ("***** OMORACLE ***** At tryResume\n"); +finalize_it: ENDtryResume static rsRetVal startSession(instanceData* pData, char* connection, char* user, @@ -170,45 +183,50 @@ finalize_it: BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature -/* Right now, this module is compatible with nothing. */ -dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); -iRet = RS_RET_INCOMPATIBLE; + /* Right now, this module is compatible with nothing. */ + dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); + iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct -char connection_string[MAXHOSTNAMELEN]; -char user[_DB_MAXUNAMELEN]; -char pwd[_DB_MAXPWDLEN]; - + char user[_DB_MAXUNAMELEN]; + char pwd[_DB_MAXPWDLEN]; + char connection_string[MAXHOSTNAMELEN]; CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); -if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { - ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); -} - - -p += sizeof ":omoracle:" - 1; - -if (*p == '\0' || *p == ',') { - dbgprintf ("Wrong char processing module arguments: %c\n", *p); - ABORT_FINALIZE(RS_RET_INVALID_PARAMS); -} + if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } -CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); -CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); -CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); -p--; -CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, - OMSR_RQD_TPL_OPT_SQL, " StdFmt")); -CHKiRet(createInstance(&pData)); -CHKiRet(startSession(pData, connection_string, user, pwd)); + p += sizeof ":omoracle:" - 1; -dbgprintf ("omoracle module got all its resources allocated " - "and connected to the DB\n"); + if (*p == '\0' || *p == ',') { + errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); + CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); + CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); + p--; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, + OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + CHKiRet(createInstance(&pData)); + pData->connection = strdup(connection_string); + if (pData->connection == NULL) { + iRet = RS_RET_OUT_OF_MEMORY; + goto finalize_it; + } + CHKiRet(startSession(pData, connection_string, user, pwd)); + + dbgprintf ("omoracle module got all its resources allocated " + "and connected to the DB\n"); + + memset(user, 0, sizeof user); + memset(pwd, 0, sizeof pwd); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -225,6 +243,9 @@ CODESTARTdoAction CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); finalize_it: + dbgprintf ("omoracle %s at executing statement %s\n", + iRet?"did not succeed":"succeeded", *ppString); +/* Clean credentials to avoid leakage in case of core dump. */ ENDdoAction BEGINmodExit @@ -233,16 +254,14 @@ ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo -dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); - + dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES -dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); - + dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); ENDqueryEtryPt static rsRetVal @@ -256,17 +275,10 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; -char dbname[MAX_BUFSIZE]; CODEmodInit_QueryRegCFSLineHdlr -dbgprintf ("***** OMORACLE ***** At modInit\n"); -CHKiRet(objUse(errmsg, CORE_COMPONENT)); -/* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ -CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, - eCmdHdlrCustomHandler, resetConfigVariables, - NULL, STD_LOADABLE_MODULE_ID)); -dbgprintf ("***** OMORACLE ***** dbname before = %s\n", dbname); -CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, - NULL, dbname, STD_LOADABLE_MODULE_ID)); -dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ + CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, + eCmdHdlrCustomHandler, resetConfigVariables, + NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit - -- cgit From 0289fb7f5c63ee1563eb829b5b1a0fcf3dfa279a Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 25 Mar 2009 18:16:31 +0100 Subject: Remove useless dbgprintf and add documentation. --- plugins/omoracle/omoracle.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index aa506dca..4cf4c724 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -43,13 +43,21 @@ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) typedef struct _instanceData { + /* Environment handler, the base for any OCI work. */ OCIEnv* environment; + /* Session handler, the actual DB connection object. */ OCISession* session; + /* Error handler for OCI calls. */ OCIError* error; + /* Prepared statement. */ OCIStmt* statement; + /* Service handler. */ OCISvcCtx* service; + /* Credentials object for the connection. */ OCIAuthInfo* authinfo; + /* Binding parameters, currently unused */ OCIBind* binding; + /* Connection string, kept here for possible retries. */ char* connection; } instanceData; @@ -177,6 +185,8 @@ static rsRetVal startSession(instanceData* pData, char* connection, char* user, strlen(connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); finalize_it: + if (iRet != RS_RET_OK) + errmsg.LogError(0, NO_ERRCODE, "Unable to start Oracle session\n"); RETiRet; } @@ -254,14 +264,12 @@ ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES - dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); ENDqueryEtryPt static rsRetVal -- cgit From c54de8212fe434080983b70d9ffa200d52f68ccd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 25 Mar 2009 19:41:17 +0100 Subject: added some (hopefully helpful) comments on the calling IF --- plugins/omoracle/omoracle.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 4cf4c724..eba45c5b 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -154,6 +154,26 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + /* Here usually only a reconnect is done. The rsyslog core will call + * this entry point from time to time when the action suspended itself. + * Note that the rsyslog core expects that if the plugin suspended itself + * the action was not carried out during that invocation. Thus, rsyslog + * will call the action with *the same* data item again AFTER a resume + * was successful. As such, tryResume should NOT write the failed data + * item. If it needs to for some reason, it must delete the item again, + * otherwise, it will get duplicated. + * This handling inside the rsyslog core is important to be able to + * preserve data over rsyslog restarts. With it, the core can ensure that + * it queues all not-yet-processed messages without the plugin needing + * to take care about that. + * So in essence, it is recommended that just a reconnet is tried, but + * the last action not restarted. Note that it is not a real problem + * (but causes a slight performance degradation) if tryResume returns + * successfully but the next call to doAction() immediately returns + * RS_RET_SUSPENDED. So it is OK to do the actual restart inside doAction(). + * ... of course I don't know why Oracle might need a full restart... + * rgerhards, 2009-03-26 + */ dbgprintf("Attempting to restart the last action\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); @@ -213,6 +233,13 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); p += sizeof ":omoracle:" - 1; + /* while this parameter parsing is convenient and works perfectly, + * it is suggested that parameters are only specified via $Action... config + * statement (as done in omlibdbi). The reason is that this may greatly + * ease the transition when we have the full config script language. However, + * this approach here is guranteed to continue to work in the future. + * rgerhards, 2009-03-26 + */ if (*p == '\0' || *p == ',') { errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); -- cgit From 59d4a52c280c00bccde4be0321bb09665cc11d29 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 16:31:41 +0200 Subject: initial work on omprog, an output module to send messages to another program --- Makefile.am | 6 +- configure.ac | 16 ++++ doc/features.html | 1 - plugins/omdtn/Makefile.am | 8 -- plugins/omdtn/omdtn.c | 130 ----------------------------- plugins/omprog/Makefile.am | 8 ++ plugins/omprog/omprog.c | 204 +++++++++++++++++++++++++++++++++++++++++++++ runtime/conf.c | 1 - runtime/rsyslog.h | 2 + 9 files changed, 235 insertions(+), 141 deletions(-) delete mode 100644 plugins/omdtn/Makefile.am delete mode 100644 plugins/omdtn/omdtn.c create mode 100644 plugins/omprog/Makefile.am create mode 100644 plugins/omprog/omprog.c diff --git a/Makefile.am b/Makefile.am index 97f4aed3..02444d6f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -108,6 +108,10 @@ if ENABLE_MAIL SUBDIRS += plugins/ommail endif +if ENABLE_OMPROG +SUBDIRS += plugins/omprog +endif + if ENABLE_RFC3195 SUBDIRS += plugins/im3195 endif @@ -122,5 +126,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index de328f83..e9e68cb4 100644 --- a/configure.ac +++ b/configure.ac @@ -668,6 +668,20 @@ AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to serach for imtemplate to find everything! +# settings for the omprog output module +AC_ARG_ENABLE(omprog, + [AS_HELP_STRING([--enable-omprog],[Compiles omprog template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omprog="yes" ;; + no) enable_omprog="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omprog) ;; + esac], + [enable_omprog=no] +) +AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) +# end of omprog + + # settings for the template output module; copy and modify this code # if you intend to add your own module. Be sure to replace omtemplate # by the actual name of your module. @@ -720,6 +734,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ plugins/omtemplate/Makefile \ + plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ @@ -752,6 +767,7 @@ echo "imdiag enabled: $enable_imdiag" echo "file input module enabled: $enable_imfile" echo "input template module will be compiled: $enable_imtemplate" echo "output template module will be compiled: $enable_omtemplate" +echo "omprog module will be compiled: $enable_omprog" echo "omstdout module will be compiled: $enable_omstdout" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" diff --git a/doc/features.html b/doc/features.html index 501f3304..336b31cc 100644 --- a/doc/features.html +++ b/doc/features.html @@ -124,7 +124,6 @@ community. Plus, it can be financially attractive: just think about how much les be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit of being recognised as a sponsor may even drive new customers to your business!
    -
  • Finalize the DTN "planetary Internet" space ship mode output plugin
  • port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge
  • pcre filtering - maybe (depending on feedback)  - diff --git a/plugins/omdtn/Makefile.am b/plugins/omdtn/Makefile.am deleted file mode 100644 index afb57476..00000000 --- a/plugins/omdtn/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -pkglib_LTLIBRARIES = omdtn.la - -omdtn_la_SOURCES = omdtn.c -omdtn_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -omdtn_la_LDFLAGS = -module -avoid-version -omdtn_la_LIBADD = - -EXTRA_DIST = diff --git a/plugins/omdtn/omdtn.c b/plugins/omdtn/omdtn.c deleted file mode 100644 index 761bde79..00000000 --- a/plugins/omdtn/omdtn.c +++ /dev/null @@ -1,130 +0,0 @@ -/* omdtn.c - * This is the plugin for rsyslog use in the interplanetary Internet, - * especially useful for rsyslog in space ships of all kinds. - * The core idea was introduced in early 2009 and considered - * doable. - * - * Note that this has not yet been tested for robustness but needs - * to prior to placing it on top of a rocket. - * - * NOTE: read comments in module-template.h for more specifics! - * - * File begun on 2009-04-01 by RGerhards - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include "rsyslog.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "dirty.h" -#include "syslogd-types.h" -#include "srUtils.h" -#include "template.h" -#include "module-template.h" -#include "errmsg.h" -#include "cfsysline.h" - -MODULE_TYPE_OUTPUT - -/* internal structures - */ -DEF_OMOD_STATIC_DATA - -typedef struct _instanceData { -} instanceData; - -BEGINcreateInstance -CODESTARTcreateInstance -ENDcreateInstance - - -BEGINisCompatibleWithFeature -CODESTARTisCompatibleWithFeature - if(eFeat == sFEATURERepeatedMsgReduction) - iRet = RS_RET_OK; -ENDisCompatibleWithFeature - - -BEGINfreeInstance -CODESTARTfreeInstance -ENDfreeInstance - - -BEGINdbgPrintInstInfo -CODESTARTdbgPrintInstInfo -ENDdbgPrintInstInfo - - -BEGINtryResume -CODESTARTtryResume -ENDtryResume - -BEGINdoAction -CODESTARTdoAction - write(1, (char*)ppString[0], strlen((char*)ppString[0])); -ENDdoAction - - -BEGINparseSelectorAct -CODESTARTparseSelectorAct -CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* first check if this config line is actually for us */ - if(strncmp((char*) p, ":omstdout:", sizeof(":omstdout:") - 1)) { - ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); - } - - /* ok, if we reach this point, we have something for us */ - p += sizeof(":omstdout:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ - CHKiRet(createInstance(&pData)); - - /* check if a non-standard template is to be applied */ - if(*(p-1) == ';') - --p; - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); -CODE_STD_FINALIZERparseSelectorAct -ENDparseSelectorAct - - -BEGINmodExit -CODESTARTmodExit -ENDmodExit - - -BEGINqueryEtryPt -CODESTARTqueryEtryPt -CODEqueryEtryPt_STD_OMOD_QUERIES -ENDqueryEtryPt - - -BEGINmodInit() -CODESTARTmodInit - *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ -CODEmodInit_QueryRegCFSLineHdlr -ENDmodInit - -/* vi:set ai: - */ diff --git a/plugins/omprog/Makefile.am b/plugins/omprog/Makefile.am new file mode 100644 index 00000000..63fe09b8 --- /dev/null +++ b/plugins/omprog/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omprog.la + +omprog_la_SOURCES = omprog.c +omprog_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omprog_la_LDFLAGS = -module -avoid-version +omprog_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c new file mode 100644 index 00000000..0cdacf78 --- /dev/null +++ b/plugins/omprog/omprog.c @@ -0,0 +1,204 @@ +/* omprog.c + * This output plugin enables rsyslog to execute a program and + * feed it the message stream as standard input. + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-04-01 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +typedef struct _instanceData { + uchar *szBinary; /* name of binary to call */ + pid_t pid; /* pid of currently running process */ + int fdPipe; /* file descriptor to write to */ + int bIsRunning; /* is binary currently running? 0-no, 1-yes */ +} instanceData; + +/* config settings */ +static uchar *szBinary = NULL; /* name of binary to call */ + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +/* creates a pipe and starts program, uses pipe as stdin for program. + * rgerhards, 2009-04-01 + */ +static rsRetVal +openPipe(instanceData *pData) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = { NULL }; + char *newenviron[] = { NULL }; + DEFiRet; + + assert(pData != NULL); + + if(pipe(pipefd) == -1) { + ABORT_FINALIZE(RS_RET_ERR_CREAT_PIPE); + } + + cpid = fork(); + if(cpid == -1) { + ABORT_FINALIZE(RS_RET_ERR_FORK); + } + + if(cpid == 0) { + /* we are now the child, just set the right selectors and + * exec the binary. If that fails, there is not much we can do. + */ + fclose(stdin); + dup(pipefd[0]); + close(pipefd[1]); + //fclose(stdout); +fprintf(stderr, "Program to exec '%s', fdPipe: %d\n", pData->szBinary, pipefd[0]); + execve((char*)pData->szBinary, newargv, newenviron); + } + + pData->fdPipe = pipefd[1]; + pData->pid = cpid; + close(pipefd[0]); + pData->bIsRunning = 1; +finalize_it: + RETiRet; +} + + +BEGINdoAction +CODESTARTdoAction + if(pData->bIsRunning == 0) { + openPipe(pData); + } + + write(pData->fdPipe, (char*)ppString[0], strlen((char*)ppString[0])); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omprog:", sizeof(":omprog:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omprog:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + CHKmalloc(pData->szBinary = (uchar*) strdup((char*)szBinary)); + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit + if(szBinary != NULL) { + free(szBinary); + szBinary = NULL; + } +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + + if(szBinary != NULL) { + free(szBinary); + szBinary = NULL; + } + + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomprogbinary", 0, eCmdHdlrGetWord, NULL, &szBinary, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit + +/* vi:set ai: + */ diff --git a/runtime/conf.c b/runtime/conf.c index ede15cc7..27ab8bb4 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -796,7 +796,6 @@ dbgprintf("calling expression parser, pp %p ('%s')\n", *pline, *pline); /* debug support - print vmprg after construction (uncomment to use) */ /* vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); */ - vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); /* we now need to skip whitespace to the action part, else we confuse * the legacy rsyslog conf parser. -- rgerhards, 2008-02-25 diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 899f5e13..2aa1868b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -258,6 +258,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ + RS_RET_ERR_CREAT_PIPE = -2114, /**< error during pipe creation */ + RS_RET_ERR_FORK = -2115, /**< error during fork() */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From 01f2c7a7a3ee394028eb36d9709490cbc26c7369 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 18:17:20 +0200 Subject: improved omprog, now ready for first practical testing --- ChangeLog | 3 + plugins/omprog/omprog.c | 169 +++++++++++++++++++++++++++++++++++++++++++++--- runtime/rsyslog.h | 1 + 3 files changed, 165 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 23a2e3e0..77ebdf83 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- new feature: new output plugin omprog, which permits to start program + and feed it (via its stdin) with syslog messages. If the program + terminates, it is restarted. - bugfix: potential abort with DA queue after high watermark is reached There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 0cdacf78..2a078a6d 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "dirty.h" #include "syslogd-types.h" #include "srUtils.h" @@ -48,6 +49,7 @@ MODULE_TYPE_OUTPUT /* internal structures */ DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) typedef struct _instanceData { uchar *szBinary; /* name of binary to call */ @@ -73,6 +75,8 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance + if(pData->szBinary != NULL) + free(pData->szBinary); ENDfreeInstance @@ -85,6 +89,52 @@ BEGINtryResume CODESTARTtryResume ENDtryResume + +/* execute the child process (must be called in child context + * after fork). + */ + +static void execBinary(instanceData *pData, int fdStdin) +{ + int i; + struct sigaction sigAct; + char *newargv[] = { NULL }; + char *newenviron[] = { NULL }; + + assert(pData != NULL); + + fclose(stdin); + dup(fdStdin); + //fclose(stdout); + + /* we close all file handles as we fork soon + * Is there a better way to do this? - mail me! rgerhards@adiscon.com + */ +# ifndef VALGRIND /* we can not use this with valgrind - too many errors... */ + for(i = 3 ; i <= 65535 ; ++i) + close(i); +# endif + + /* reset signal handlers to default */ + memset(&sigAct, 0, sizeof(sigAct)); + sigfillset(&sigAct.sa_mask); + sigAct.sa_handler = SIG_DFL; + for(i = 1 ; i < NSIG ; ++i) + sigaction(i, &sigAct, NULL); + + alarm(0); + + /* finally exec child */ + execve((char*)pData->szBinary, newargv, newenviron); + /* switch to? + execlp((char*)program, (char*) program, (char*)arg, NULL); + */ + + /* we should never reach this point, but if we do, we terminate */ + exit(1); +} + + /* creates a pipe and starts program, uses pipe as stdin for program. * rgerhards, 2009-04-01 */ @@ -93,8 +143,6 @@ openPipe(instanceData *pData) { int pipefd[2]; pid_t cpid; - char *newargv[] = { NULL }; - char *newenviron[] = { NULL }; DEFiRet; assert(pData != NULL); @@ -103,6 +151,10 @@ openPipe(instanceData *pData) ABORT_FINALIZE(RS_RET_ERR_CREAT_PIPE); } + DBGPRINTF("executing program '%s'\n", pData->szBinary); + + /* NO OUTPUT AFTER FORK! */ + cpid = fork(); if(cpid == -1) { ABORT_FINALIZE(RS_RET_ERR_FORK); @@ -112,14 +164,12 @@ openPipe(instanceData *pData) /* we are now the child, just set the right selectors and * exec the binary. If that fails, there is not much we can do. */ - fclose(stdin); - dup(pipefd[0]); close(pipefd[1]); - //fclose(stdout); -fprintf(stderr, "Program to exec '%s', fdPipe: %d\n", pData->szBinary, pipefd[0]); - execve((char*)pData->szBinary, newargv, newenviron); + execBinary(pData, pipefd[0]); + /*NO CODE HERE - WILL NEVER BE REACHED!*/ } + DBGPRINTF("child has pid %d\n", cpid); pData->fdPipe = pipefd[1]; pData->pid = cpid; close(pipefd[0]); @@ -129,13 +179,113 @@ finalize_it: } +/* clean up after a terminated child + */ +static inline rsRetVal +cleanup(instanceData *pData) +{ + int status; + int ret; + char errStr[1024]; + DEFiRet; + + assert(pData != NULL); + assert(pData->bIsRunning == 1); +RUNLOG_VAR("%d", pData->pid); + ret = waitpid(pData->pid, &status, 0); + if(ret != pData->pid) { + /* if waitpid() fails, we can not do much - try to ignore it... */ + DBGPRINTF("waitpid() returned state %d[%s], future malfunction may happen\n", ret, + rs_strerror_r(errno, errStr, sizeof(errStr))); + } else { + /* check if we should print out some diagnostic information */ + DBGPRINTF("waitpid status return for program '%s': %2.2x\n", + pData->szBinary, status); + if(WIFEXITED(status)) { + errmsg.LogError(0, NO_ERRCODE, "program '%s' exited normally, state %d", + pData->szBinary, WEXITSTATUS(status)); + } else if(WIFSIGNALED(status)) { + errmsg.LogError(0, NO_ERRCODE, "program '%s' terminated by signal %d.", + pData->szBinary, WTERMSIG(status)); + } + } + + pData->bIsRunning = 0; + RETiRet; +} + + +/* try to restart the binary when it has stopped. + */ +static inline rsRetVal +tryRestart(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + assert(pData->bIsRunning == 0); + + iRet = openPipe(pData); + RETiRet; +} + + +/* write to pipe + * note that we do not try to run block-free. If the users fears something + * may block (and this not be acceptable), the action should be run on its + * own action queue. + */ +static rsRetVal +writePipe(instanceData *pData, uchar *szMsg) +{ + int lenWritten; + int lenWrite; + int writeOffset; + char errStr[1024]; + DEFiRet; + + assert(pData != NULL); + + lenWrite = strlen((char*)szMsg); + writeOffset = 0; + + do + { + lenWritten = write(pData->fdPipe, ((char*)szMsg)+writeOffset, lenWrite); + if(lenWritten == -1) { + switch(errno) { + case EPIPE: + DBGPRINTF("Program '%s' terminated, trying to restart\n", + pData->szBinary); + CHKiRet(cleanup(pData)); + CHKiRet(tryRestart(pData)); + break; + default: + DBGPRINTF("error %d writing to pipe: %s\n", errno, + rs_strerror_r(errno, errStr, sizeof(errStr))); + ABORT_FINALIZE(RS_RET_ERR_WRITE_PIPE); + break; + } + } else { + writeOffset += lenWritten; + } + } while(lenWritten != lenWrite); + + +finalize_it: + RETiRet; +} + + BEGINdoAction CODESTARTdoAction if(pData->bIsRunning == 0) { openPipe(pData); } + + iRet = writePipe(pData, ppString[0]); - write(pData->fdPipe, (char*)ppString[0], strlen((char*)ppString[0])); + if(iRet != RS_RET_OK) + iRet = RS_RET_SUSPENDED; ENDdoAction @@ -166,6 +316,8 @@ CODESTARTmodExit free(szBinary); szBinary = NULL; } + CHKiRet(objRelease(errmsg, CORE_COMPONENT)); +finalize_it: ENDmodExit @@ -195,6 +347,7 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomprogbinary", 0, eCmdHdlrGetWord, NULL, &szBinary, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 2aa1868b..ac9b726f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -260,6 +260,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_ERR_CREAT_PIPE = -2114, /**< error during pipe creation */ RS_RET_ERR_FORK = -2115, /**< error during fork() */ + RS_RET_ERR_WRITE_PIPE = -2116, /**< error writing to pipe */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From d702d3f6ff1540691aae29012dd142212e0eb04c Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Mon, 30 Mar 2009 10:39:36 +0200 Subject: Make tryResume not to retry the last action, but just to reconnect. The core will call the action if tryResume succeeds, no need to make it from here. --- plugins/omoracle/omoracle.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index eba45c5b..29ec2303 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -174,7 +174,7 @@ CODESTARTtryResume * ... of course I don't know why Oracle might need a full restart... * rgerhards, 2009-03-26 */ - dbgprintf("Attempting to restart the last action\n"); + dbgprintf("Attempting to reconnect to DB server\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, @@ -182,9 +182,6 @@ CODESTARTtryResume pData->connection, strlen(pData->connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); - CHECKERR(pData->error, OCIStmtExecute(pData->service, pData->statement, - pData->error, 1, 0, NULL, NULL, - OCI_DEFAULT)); finalize_it: ENDtryResume -- cgit From 56bf679723f2821d0f66339b95d847f1e4ddc17b Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Mon, 30 Mar 2009 10:39:37 +0200 Subject: Add a SELinux policy that allows the module to load on RHEL5. --- plugins/omoracle/omoracle.te | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 plugins/omoracle/omoracle.te diff --git a/plugins/omoracle/omoracle.te b/plugins/omoracle/omoracle.te new file mode 100644 index 00000000..81eb6cf1 --- /dev/null +++ b/plugins/omoracle/omoracle.te @@ -0,0 +1,13 @@ + +module omoracle 1.0; + +require { + type syslogd_t; + type port_t; + class process { execstack execmem }; + class tcp_socket name_connect; +} + +#============= syslogd_t ============== +allow syslogd_t port_t:tcp_socket name_connect; +allow syslogd_t self:process { execstack execmem }; -- cgit From 70b8624c86e7d204d7c1ff91d030ee7c69569eb7 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 1 Apr 2009 18:12:46 +0200 Subject: Convert the module configuration to $Action... directives. Instead of using the old-style configuration parameters, use $... directives, which lead to simpler code, and also should make user's configurations simpler. Needs some testing. Currently, the supported directives are $OmoracleDB, $OmoracleDBUser and $OmoracleDBPassword. $OmoracleDBStatement and $OmoracleDBBatchSize may follow. --- plugins/omoracle/omoracle.c | 66 +++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 29ec2303..6ec023e2 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -4,10 +4,18 @@ database. It uses Oracle Call Interface, a propietary module provided by Oracle. - Config lines to be used are of this form: + Selector lines to be used are of this form: - :omoracle:dbstring,user,password;StatementTemplate + :omoracle:;TemplateName + The module gets its configuration via rsyslog $... directives, + namely: + + $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db + name as specified by tnsnames.ora) + All fields are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -61,6 +69,14 @@ typedef struct _instanceData { char* connection; } instanceData; +/** Database name, to be filled by the $OmoracleDB directive */ +static char* db_name; +/** Database user name, to be filled by the $OmoracleDBUser + * directive */ +static char* db_user; +/** Database password, to be filled by the $OmoracleDBPassword */ +static char* db_password; + /** Generic function for handling errors from OCI. It will be called only inside CHECKERR and CHECKENV macros. @@ -203,7 +219,8 @@ static rsRetVal startSession(instanceData* pData, char* connection, char* user, OCI_DEFAULT)); finalize_it: if (iRet != RS_RET_OK) - errmsg.LogError(0, NO_ERRCODE, "Unable to start Oracle session\n"); + errmsg.LogError(0, NO_ERRCODE, + "Unable to start Oracle session\n"); RETiRet; } @@ -217,10 +234,6 @@ ENDisCompatibleWithFeature BEGINparseSelectorAct - char user[_DB_MAXUNAMELEN]; - char pwd[_DB_MAXPWDLEN]; - char connection_string[MAXHOSTNAMELEN]; - CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -230,37 +243,26 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); p += sizeof ":omoracle:" - 1; - /* while this parameter parsing is convenient and works perfectly, - * it is suggested that parameters are only specified via $Action... config - * statement (as done in omlibdbi). The reason is that this may greatly - * ease the transition when we have the full config script language. However, - * this approach here is guranteed to continue to work in the future. - * rgerhards, 2009-03-26 - */ if (*p == '\0' || *p == ',') { - errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); + errmsg.LogError(0, NO_ERRCODE, + "Wrong char processing module arguments: %c\n", + *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } - CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); - CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); - CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); - p--; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); CHKiRet(createInstance(&pData)); - pData->connection = strdup(connection_string); + pData->connection = strdup(db_name); if (pData->connection == NULL) { iRet = RS_RET_OUT_OF_MEMORY; goto finalize_it; } - CHKiRet(startSession(pData, connection_string, user, pwd)); + CHKiRet(startSession(pData, db_name, db_user, db_password)); dbgprintf ("omoracle module got all its resources allocated " "and connected to the DB\n"); - memset(user, 0, sizeof user); - memset(pwd, 0, sizeof pwd); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -300,7 +302,16 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { + int n; DEFiRet; + free(db_user); + free(db_name); + if (db_password != NULL) { + n = strlen(db_password); + memset(db_password, n, sizeof *db_password); + free(db_password); + } + db_name = db_user = db_password = NULL; RETiRet; } @@ -313,4 +324,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbuser", 0, + eCmdHdlrGetWord, NULL, &db_user, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbpassword", 0, + eCmdHdlrGetWord, NULL, &db_password, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, + eCmdHdlrGetWord, NULL, &db_name, + STD_LOADABLE_MODULE_ID)); ENDmodInit -- cgit From 8a819d6a0623c0854462dd2dc632700f9a5c576c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 18:32:18 +0200 Subject: some small changes (as suggestion) --- plugins/omoracle/omoracle.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 6ec023e2..ea910d3a 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -253,11 +253,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); CHKiRet(createInstance(&pData)); - pData->connection = strdup(db_name); - if (pData->connection == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto finalize_it; - } + CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); dbgprintf ("omoracle module got all its resources allocated " @@ -304,11 +300,13 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, { int n; DEFiRet; - free(db_user); - free(db_name); + if(db_user != NULL) + free(db_user); + if(db_name != NULL) + free(db_name); if (db_password != NULL) { n = strlen(db_password); - memset(db_password, n, sizeof *db_password); + memset(db_password, 0, n); free(db_password); } db_name = db_user = db_password = NULL; -- cgit From 8e3c5a9ca3732a41fbb8581b13c49acd699820da Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 17:55:04 +0200 Subject: improved internal handling of RainerScript functions - building the necessary plumbing to support more functions with decent runtime performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" --- ChangeLog | 7 ++ runtime/expr.c | 4 +- runtime/rsyslog.h | 4 + runtime/vm.c | 252 +++++++++++++++++++++++++++++++++++++++++++++++++++--- runtime/vm.h | 5 +- runtime/vmop.c | 53 ++++++++++-- runtime/vmop.h | 11 ++- runtime/vmprg.c | 28 +++++- runtime/vmprg.h | 4 +- runtime/vmstk.h | 4 +- tests/3.rstest | 4 +- tools/syslogd.c | 1 + 12 files changed, 345 insertions(+), 32 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8d4191d3..24898dc7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +--------------------------------------------------------------------------- +Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? +- improved internal handling of RainerScript functions, building the + necessary plumbing to support more functions with decent runtime + performance. This is also necessary towards the long-term goal + of loadable library modules. +- added new RainerScript function "tolower" - bugfix: potential abort with DA queue after high watermark is reached There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which diff --git a/runtime/expr.c b/runtime/expr.c index 38ed1c68..e449d1c7 100644 --- a/runtime/expr.c +++ b/runtime/expr.c @@ -142,7 +142,9 @@ terminal(expr_t *pThis, ctok_t *tok) * we have all relevant information) */ CHKiRet(ctok_token.UnlinkVar(pToken, &pVar)); - CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_FUNC_CALL, pVar)); /* add to program */ + CHKiRet(var.ConvToString(pVar)); /* make sure we have a string */ + CHKiRet(vmprg.AddCallOperation(pThis->pVmprg, pVar->val.pStr)); /* add to program */ + CHKiRet(var.Destruct(&pVar)); break; case ctok_MSGVAR: dbgoprint((obj_t*) pThis, "MSGVAR\n"); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 032d8c04..cea457d8 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -84,6 +84,8 @@ typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; +typedef struct vmstk_s vmstk_t; +typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ /* some universal 64 bit define... */ typedef long long int64; @@ -258,6 +260,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ + RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ + RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/vm.c b/runtime/vm.c index a25476c2..41d3e483 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -26,6 +26,7 @@ #include "config.h" #include #include +#include #include "rsyslog.h" #include "obj.h" @@ -39,6 +40,142 @@ DEFobjCurrIf(vmstk) DEFobjCurrIf(var) DEFobjCurrIf(sysvar) +/* ------------------------------ function registry code and structures ------------------------------ */ + +/* we maintain a registry of known functions */ +/* currently, this is a singly-linked list, this shall become a binary + * tree when we add the real call interface. So far, entries are added + * at the root, only. + */ +typedef struct s_rsf_entry { + cstr_t *pName; /* function name */ + prsf_t rsf; /* pointer to function code */ + struct s_rsf_entry *pNext; /* Pointer to next element or NULL */ +} rsf_entry_t; +rsf_entry_t *funcRegRoot = NULL; + + +/* add a function to the function registry. + * The handed-over cstr_t* object must no longer be used by the caller. + * A duplicate function name is an error. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsfrAddFunction(uchar *szName, prsf_t rsf) +{ + rsf_entry_t *pEntry; + size_t lenName; + DEFiRet; + + assert(szName != NULL); + assert(rsf != NULL); + + /* first check if we have a duplicate name, with the current approach this means + * we need to go through the whole list. + */ + lenName = strlen((char*)szName); + for(pEntry = funcRegRoot ; pEntry != NULL ; pEntry = pEntry->pNext) + if(!rsCStrSzStrCmp(pEntry->pName, szName, lenName)) + ABORT_FINALIZE(RS_RET_DUP_FUNC_NAME); + + /* unique name, so add to head of list */ + CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); + CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + pEntry->rsf = rsf; + pEntry->pNext = funcRegRoot; + funcRegRoot = pEntry; + +finalize_it: + RETiRet; +} + + +/* find a function inside the function registry + * The caller provides a cstr_t with the function name and receives + * a function pointer back. If no function is found, an RS_RET_UNKNW_FUNC + * error is returned. So if the function returns with RS_RET_OK, the caller + * can savely assume the function pointer is valid. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunction(cstr_t *pcsName, prsf_t *prsf) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(prsf != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(!rsCStrCStrCmp(pEntry->pName, pcsName)) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *prsf = pFound->rsf; + +finalize_it: + RETiRet; +} + + +/* find the name of a RainerScript function whom's function pointer + * is known. This function returns the cstr_t object, which MUST NOT + * be modified by the caller. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunctionName(prsf_t rsf, cstr_t **ppcsName) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(rsf != NULL); + assert(ppcsName != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(pEntry->rsf == rsf) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *ppcsName = pFound->pName; + +finalize_it: + RETiRet; +} + + +/* free the whole function registry + */ +static void +rsfrRemoveAll(void) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pEntryDel; + + BEGINfunc + pEntry = funcRegRoot; + while(pEntry != NULL) { + pEntryDel = pEntry; + pEntry = pEntry->pNext; + rsCStrDestruct(&pEntryDel->pName); + free(pEntryDel); + } + funcRegRoot = NULL; + ENDfunc +} + + +/* ------------------------------ end function registry code and structures ------------------------------ */ + /* ------------------------------ instruction set implementation ------------------------------ * * The following functions implement the VM's instruction set. @@ -330,7 +467,6 @@ CODESTARTop(PUSHSYSVAR) finalize_it: ENDop(PUSHSYSVAR) - /* The function call operation is only very roughly implemented. While the plumbing * to reach this instruction is fine, the instruction itself currently supports only * functions with a single argument AND with a name that we know. @@ -340,20 +476,9 @@ ENDop(PUSHSYSVAR) */ BEGINop(FUNC_CALL) /* remember to set the instruction also in the ENDop macro! */ var_t *numOperands; - var_t *operand1; - int iStrlen; CODESTARTop(FUNC_CALL) vmstk.PopNumber(pThis->pStk, &numOperands); - if(numOperands->val.num != 1) - ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); - vmstk.PopString(pThis->pStk, &operand1); /* guess there's just one ;) */ - if(!rsCStrSzStrCmp(pOp->operand.pVar->val.pStr, (uchar*) "strlen", 6)) { /* only one supported so far ;) */ -RUNLOG_VAR("%s", rsCStrGetSzStr(operand1->val.pStr)); - iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); -RUNLOG_VAR("%d", iStrlen); - } else - ABORT_FINALIZE(RS_RET_INVLD_FUNC); - PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME + CHKiRet((*pOp->operand.rsf)(pThis->pStk, numOperands->val.num)); var.Destruct(&numOperands); /* no longer needed */ finalize_it: ENDop(FUNC_CALL) @@ -362,6 +487,89 @@ ENDop(FUNC_CALL) /* ------------------------------ end instruction set implementation ------------------------------ */ +/* ------------------------------ begin built-in function implementation ------------------------------ */ +/* note: this shall probably be moved to a separate module, but for the time being we do it directly + * in here. This is on our way to get from a dirty to a clean solution via baby steps that are + * a bit less dirty each time... + * + * The advantage of doing it here is that we do not yet need to think about how to handle the + * exit case, where we must not unload function modules which functions are still referenced. + * + * CALLING INTERFACE: + * The function must pop its parameters off the stack and pop its result onto + * the stack when it is finished. The number of parameters the function was + * called with is provided to it. If the argument count is less then what the function + * expected, it may handle the situation with defaults (or return an error). If the + * argument count is greater than expected, returnung an error is highly + * recommended (use RS_RET_INVLD_NBR_ARGUMENTS for these cases). + * + * All function names are prefixed with "rsf_" (RainerScript Function) to have + * a separate "name space". + * + * rgerhards, 2009-04-06 + */ + + +/* The strlen function, also probably a prototype of how all functions should be + * implemented. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_strlen(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton (trivial case here...) */ + vmstk.PopString(pStk, &operand1); + iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); + + /* Store result and cleanup */ + var.SetNumber(operand1, iStrlen); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + +/* The "tolower" function, which converts its sole argument to lower case. + * Quite honestly, currently this is primarily a test driver for me... + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_tolower(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + uchar *pSrc; + cstr_t *pcstr; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton */ + CHKiRet(rsCStrConstruct(&pcstr)); + vmstk.PopString(pStk, &operand1); + pSrc = rsCStrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); + while(iStrlen--) { + CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + } + + /* Store result and cleanup */ + CHKiRet(rsCStrFinish(pcstr)); + var.SetString(operand1, pcstr); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(vm) /* be sure to specify the object type also in END macro! */ @@ -531,10 +739,23 @@ CODESTARTobjQueryInterface(vm) pIf->PopBoolFromStack = PopBoolFromStack; pIf->PopVarFromStack = PopVarFromStack; pIf->SetMsg = SetMsg; + pIf->FindRSFunction = findRSFunction; + pIf->FindRSFunctionName = findRSFunctionName; finalize_it: ENDobjQueryInterface(vm) +/* Exit the vm class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(vm, OBJ_IS_CORE_MODULE) /* class, version */ + rsfrRemoveAll(); + objRelease(sysvar, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vmstk, CORE_COMPONENT); +ENDObjClassExit(vm) + + /* Initialize the vm class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-02-19 @@ -548,6 +769,11 @@ BEGINObjClassInit(vm, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, vmDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmConstructFinalize); + + /* register built-in functions // TODO: move to its own module */ + CHKiRet(rsfrAddFunction((uchar*)"strlen", rsf_strlen)); + CHKiRet(rsfrAddFunction((uchar*)"tolower", rsf_tolower)); + ENDObjClassInit(vm) /* vi:set ai: diff --git a/runtime/vm.h b/runtime/vm.h index d2458220..cb3c69d0 100644 --- a/runtime/vm.h +++ b/runtime/vm.h @@ -55,8 +55,11 @@ BEGINinterface(vm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*PopBoolFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*PopVarFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*SetMsg)(vm_t *pThis, msg_t *pMsg); /* there are a few cases where we need this... */ + /* v2 (4.1.7) */ + rsRetVal (*FindRSFunction)(cstr_t *pcsName, prsf_t *prsf); /* 2009-06-04 */ + rsRetVal (*FindRSFunctionName)(prsf_t rsf, cstr_t **ppcsName); /* 2009-06-04 */ ENDinterface(vm) -#define vmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmop.c b/runtime/vmop.c index a343481e..3e001d27 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -32,10 +32,12 @@ #include "rsyslog.h" #include "obj.h" #include "vmop.h" +#include "vm.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(var) +DEFobjCurrIf(vm) /* forward definitions */ @@ -61,8 +63,10 @@ rsRetVal vmopConstructFinalize(vmop_t __attribute__((unused)) *pThis) /* destructor for the vmop object */ BEGINobjDestruct(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(vmop) - if(pThis->operand.pVar != NULL) - var.Destruct(&pThis->operand.pVar); + if(pThis->opcode != opcode_FUNC_CALL) { + if(pThis->operand.pVar != NULL) + var.Destruct(&pThis->operand.pVar); + } ENDobjDestruct(vmop) @@ -72,13 +76,19 @@ BEGINobjDebugPrint(vmop) /* be sure to specify the object type also in END and C cstr_t *pStrVar; CODESTARTobjDebugPrint(vmop) vmopOpcode2Str(pThis, &pOpcodeName); - CHKiRet(rsCStrConstruct(&pStrVar)); - CHKiRet(rsCStrFinish(&pStrVar)); - if(pThis->operand.pVar != NULL) { - CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pStrVar)); + assert(pStrVar != NULL); + } else { + CHKiRet(rsCStrConstruct(&pStrVar)); + if(pThis->operand.pVar != NULL) { + CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + } } + CHKiRet(rsCStrFinish(&pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); - rsCStrDestruct(&pStrVar); + if(pThis->opcode != opcode_FUNC_CALL) + rsCStrDestruct(&pStrVar); finalize_it: ENDobjDebugPrint(vmop) @@ -98,6 +108,7 @@ static rsRetVal Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) { uchar *pOpcodeName; + cstr_t *pcsFuncName; uchar szBuf[2048]; size_t lenBuf; DEFiRet; @@ -107,8 +118,13 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) vmopOpcode2Str(pThis, &pOpcodeName); lenBuf = snprintf((char*) szBuf, sizeof(szBuf), "%s\t", pOpcodeName); CHKiRet(rsCStrAppendStrWithLen(pstrPrg, szBuf, lenBuf)); - if(pThis->operand.pVar != NULL) - CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pcsFuncName)); + CHKiRet(rsCStrAppendCStr(pstrPrg, pcsFuncName)); + } else { + if(pThis->operand.pVar != NULL) + CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + } CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); finalize_it: @@ -116,6 +132,23 @@ finalize_it: } +/* set function + * rgerhards, 2009-04-06 + */ +static rsRetVal +vmopSetFunc(vmop_t *pThis, cstr_t *pcsFuncName) +{ + prsf_t rsf; /* pointer to function */ + DEFiRet; + ISOBJ_TYPE_assert(pThis, vmop); + CHKiRet(vm.FindRSFunction(pcsFuncName, &rsf)); /* check if function exists and obtain pointer to it */ + assert(rsf != NULL); /* just double-check, would be very hard to find! */ + pThis->operand.rsf = rsf; +finalize_it: + RETiRet; +} + + /* set operand (variant case) * rgerhards, 2008-02-20 */ @@ -248,6 +281,7 @@ CODESTARTobjQueryInterface(vmop) pIf->ConstructFinalize = vmopConstructFinalize; pIf->Destruct = vmopDestruct; pIf->DebugPrint = vmopDebugPrint; + pIf->SetFunc = vmopSetFunc; pIf->SetOpcode = vmopSetOpcode; pIf->SetVar = vmopSetVar; pIf->Opcode2Str = vmopOpcode2Str; @@ -263,6 +297,7 @@ ENDobjQueryInterface(vmop) BEGINObjClassInit(vmop, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); OBJSetMethodHandler(objMethod_DEBUGPRINT, vmopDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmopConstructFinalize); diff --git a/runtime/vmop.h b/runtime/vmop.h index 938b08fd..67048c26 100644 --- a/runtime/vmop.h +++ b/runtime/vmop.h @@ -26,6 +26,7 @@ #define INCLUDED_VMOP_H #include "ctok_token.h" +#include "vmstk.h" #include "stringbuf.h" /* machine instructions types */ @@ -96,7 +97,8 @@ typedef struct vmop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ opcode_t opcode; union { - var_t *pVar; /* for function call, this is the name (string) of function to be called */ + var_t *pVar; + prsf_t rsf; /* pointer to function for "call" instruction */ } operand; struct vmop_s *pNext; /* next operation or NULL, if end of program (logically this belongs to vmprg) */ } vmop_t; @@ -112,8 +114,13 @@ BEGINinterface(vmop) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetVar)(vmop_t *pThis, var_t *pVar); rsRetVal (*Opcode2Str)(vmop_t *pThis, uchar **ppName); rsRetVal (*Obj2Str)(vmop_t *pThis, cstr_t *pstr); + /* v2 */ + rsRetVal (*SetFunc)(vmop_t *pThis, cstr_t *pcsFuncName); ENDinterface(vmop) -#define vmopCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmopCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* interface changes, v1 -> v2 + * added SetFuct after existing function pointers -- rgerhards, 2009-04-06 + */ /* the remaining prototypes */ PROTOTYPEObj(vmop); diff --git a/runtime/vmprg.c b/runtime/vmprg.c index 75915025..07757b98 100644 --- a/runtime/vmprg.c +++ b/runtime/vmprg.c @@ -155,7 +155,6 @@ vmprgAddVarOperation(vmprg_t *pThis, opcode_t opcode, var_t *pVar) /* construct and fill vmop */ CHKiRet(vmop.Construct(&pOp)); CHKiRet(vmop.ConstructFinalize(pOp)); - CHKiRet(vmop.ConstructFinalize(pOp)); CHKiRet(vmop.SetOpcode(pOp, opcode)); if(pVar != NULL) CHKiRet(vmop.SetVar(pOp, pVar)); @@ -168,6 +167,32 @@ finalize_it: } +/* this is another shortcut for high-level callers. It is similar to vmprgAddVarOperation + * but adds a call operation. Among others, this include a check if the function + * is known. + */ +static rsRetVal +vmprgAddCallOperation(vmprg_t *pThis, cstr_t *pcsName) +{ + DEFiRet; + vmop_t *pOp; + + ISOBJ_TYPE_assert(pThis, vmprg); + + /* construct and fill vmop */ + CHKiRet(vmop.Construct(&pOp)); + CHKiRet(vmop.ConstructFinalize(pOp)); + CHKiRet(vmop.SetFunc(pOp, pcsName)); + CHKiRet(vmop.SetOpcode(pOp, opcode_FUNC_CALL)); + + /* and add it to the program */ + CHKiRet(vmprgAddOperation(pThis, pOp)); + +finalize_it: + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-02-21 */ @@ -189,6 +214,7 @@ CODESTARTobjQueryInterface(vmprg) pIf->Obj2Str = Obj2Str; pIf->AddOperation = vmprgAddOperation; pIf->AddVarOperation = vmprgAddVarOperation; + pIf->AddCallOperation = vmprgAddCallOperation; finalize_it: ENDobjQueryInterface(vmprg) diff --git a/runtime/vmprg.h b/runtime/vmprg.h index c1042f7d..66f03913 100644 --- a/runtime/vmprg.h +++ b/runtime/vmprg.h @@ -57,8 +57,10 @@ BEGINinterface(vmprg) /* name must also be changed in ENDinterface macro! */ rsRetVal (*AddOperation)(vmprg_t *pThis, vmop_t *pOp); rsRetVal (*AddVarOperation)(vmprg_t *pThis, opcode_t opcode, var_t *pVar); rsRetVal (*Obj2Str)(vmprg_t *pThis, cstr_t *pstr); + /* v2 (4.1.7) */ + rsRetVal (*AddCallOperation)(vmprg_t *pThis, cstr_t *pVar); /* added 2009-04-06 */ ENDinterface(vmprg) -#define vmprgCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmprgCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmstk.h b/runtime/vmstk.h index 2d45ee4d..06657cf4 100644 --- a/runtime/vmstk.h +++ b/runtime/vmstk.h @@ -27,11 +27,11 @@ #define VMSTK_SIZE 256 /* the vmstk object */ -typedef struct vmstk_s { +struct vmstk_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ var_t *vStk[VMSTK_SIZE];/* the actual stack */ int iStkPtr; /* stack pointer, points to next free location, grows from 0 --> topend */ -} vmstk_t; +}; /* interfaces */ diff --git a/tests/3.rstest b/tests/3.rstest index 93cb941a..e75d9754 100644 --- a/tests/3.rstest +++ b/tests/3.rstest @@ -7,10 +7,10 @@ out: 00000000: push_msgvar msg[cstr] 00000001: push_const abc[cstr] 00000002: push_const 1[nbr] -00000003: func_call strlen[cstr] +00000003: func_call strlen 00000004: strconcat 00000005: push_const 1[nbr] -00000006: func_call strlen[cstr] +00000006: func_call strlen 00000007: push_const 20[nbr] 00000008: push_const 30[nbr] 00000009: add diff --git a/tools/syslogd.c b/tools/syslogd.c index b23c12a7..c85c4371 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3091,6 +3091,7 @@ GlobalClassExit(void) objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ objRelease(conf, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); + vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ objRelease(vm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); -- cgit From 7db9f96fe9ecb9f8e05d45cc888aa488d8aed85f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 18:07:52 +0200 Subject: testcase added (on user request) --- tests/testsuites/3.parse1 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/testsuites/3.parse1 diff --git a/tests/testsuites/3.parse1 b/tests/testsuites/3.parse1 new file mode 100644 index 00000000..a6b4e884 --- /dev/null +++ b/tests/testsuites/3.parse1 @@ -0,0 +1,3 @@ +<38>Apr 6 15:07:10 lxcvs07 sshd(pam_unix)[31738]: session closed for user cvsadmin +38,auth,info,Apr 6 15:07:10,lxcvs07,sshd(pam_unix),sshd(pam_unix)[31738]:, session closed for user cvsadmin +# yet another real-life sample where we had some issues with -- cgit From 8f8e2cd66bd7f1e35f7bf0678e25bbdb99d67093 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 12:19:54 +0200 Subject: improved testbench, added tests for tcp-based reception --- ChangeLog | 1 + tests/Makefile.am | 6 +- tests/nettester.c | 392 ++++++++++++++++++++++++++++++++++++ tests/omod-if-array.sh | 6 +- tests/parsertest.sh | 6 +- tests/testsuites/omod-if-array.conf | 3 +- tests/testsuites/parse1.conf | 3 +- tests/udptester.c | 293 --------------------------- 8 files changed, 408 insertions(+), 302 deletions(-) create mode 100644 tests/nettester.c delete mode 100644 tests/udptester.c diff --git a/ChangeLog b/ChangeLog index 821d9439..98146600 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" +- improved testbench, added tests for tcp-based reception --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/tests/Makefile.am b/tests/Makefile.am index ab1c5a62..fa662a3e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,5 +1,5 @@ TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail udptester +check_PROGRAMS = $(TESTRUNS) ourtail nettester TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid @@ -32,8 +32,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ ourtail_SOURCES = ourtail.c -udptester_SOURCES = udptester.c getline.c -udptester_LDADD = $(SOL_LIBS) +nettester_SOURCES = nettester.c getline.c +nettester_LDADD = $(SOL_LIBS) rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/nettester.c b/tests/nettester.c new file mode 100644 index 00000000..89a784f3 --- /dev/null +++ b/tests/nettester.c @@ -0,0 +1,392 @@ +/* Runs a test suite on the rsyslog (and later potentially + * other things). + * + * The name of the test suite must be given as argv[1]. In this config, + * rsyslogd is loaded with config ./testsuites/.conf and then + * test cases ./testsuites/ *. are executed on it. This test driver is + * suitable for testing cases where a message sent (via UDP) results in + * exactly one response. It can not be used in cases where no response + * is expected (that would result in a hang of the test driver). + * Note: each test suite can contain many tests, but they all need to work + * with the same rsyslog configuration. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static enum { inputUDP, inputTCP } inputMode; /* input for which tests are to be run */ +static pid_t rsyslogdPid = 0; /* pid of rsyslog instance being tested */ +static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ +static char *testSuite; /* name of current test suite */ + + +void readLine(int fd, char *ln) +{ + char c; + int lenRead; + lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + *ln++ = c; + lenRead = read(fd, &c, 1); + } + *ln = '\0'; +} + + +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + +/* send a message via UDP + * returns 0 if ok, something else otherwise. + */ +int +udpSend(char *buf, int lenBuf) +{ + struct sockaddr_in si_other; + int s, slen=sizeof(si_other); + + if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &si_other, 0, sizeof(si_other)); + si_other.sin_family = AF_INET; + si_other.sin_port = htons(12514); + if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + + if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { + perror("sendto"); + fprintf(stderr, "sendto() failed\n"); + return(1); + } + + close(s); + return 0; +} + + +/* open pipe to test candidate - so far, this is + * always rsyslogd and with a fixed config. Later, we may + * change this. Returns 0 if ok, something else otherwise. + * rgerhards, 2009-03-31 + */ +int openPipe(char *configFile, pid_t *pid, int *pfd) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", + "-M../runtime/.libs:../.libs", NULL }; + char confFile[1024]; + char *newenviron[] = { NULL }; + + + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); + newargv[1] = confFile; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if(cpid == 0) { /* Child reads from pipe */ + fclose(stdout); + dup(pipefd[1]); + close(pipefd[1]); + close(pipefd[0]); + fclose(stdin); + execve("../tools/rsyslogd", newargv, newenviron); + } else { + close(pipefd[1]); + *pid = cpid; + *pfd = pipefd[0]; + } + + return(0); +} + + +/* Process a specific test case. File name is provided. + * Needs to return 0 if all is OK, something else otherwise. + */ +int +processTestFile(int fd, char *pszFileName) +{ + FILE *fp; + char *testdata = NULL; + char *expected = NULL; + int ret = 0; + size_t lenLn; + char buf[4096]; + + if((fp = fopen((char*)pszFileName, "r")) == NULL) { + perror((char*)pszFileName); + return(2); + } + + /* skip comments at start of file */ + + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + + + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send (we could use function pointers here...) */ + if(inputMode == inputUDP) { + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + } else { + if(tcpSend(testdata, strlen(testdata)) != 0) + return(2); + } + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + free(testdata); + free(expected); + fclose(fp); + return(ret); +} + + +/* carry out all tests. Tests are specified via a file name + * wildcard. Each of the files is read and the test carried + * out. + * Returns the number of tests that failed. Zero means all + * success. + */ +int +doTests(int fd, char *files) +{ + int iFailed = 0; + int iTests = 0; + int ret; + char *testFile; + glob_t testFiles; + size_t i = 0; + struct stat fileInfo; + + glob(files, GLOB_MARK, NULL, &testFiles); + + for(i = 0; i < testFiles.gl_pathc; i++) { + testFile = testFiles.gl_pathv[i]; + + if(stat((char*) testFile, &fileInfo) != 0) + continue; /* continue with the next file if we can't stat() the file */ + + ++iTests; + /* all regular files are run through the test logic. Symlinks don't work. */ + if(S_ISREG(fileInfo.st_mode)) { /* config file */ + printf("processing test case '%s' ... ", testFile); + ret = processTestFile(fd, testFile); + if(ret == 0) { + printf("successfully completed\n"); + } else { + printf("failed!\n"); + ++iFailed; + } + } + } + globfree(&testFiles); + + if(iTests == 0) { + printf("Error: no test cases found, no tests executed.\n"); + iFailed = 1; + } else { + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + } + + return(iFailed); +} + +/* cleanup */ +void doAtExit(void) +{ + int status; + + if(rsyslogdPid != 0) { + kill(rsyslogdPid, SIGTERM); + waitpid(rsyslogdPid, &status, 0); /* wait until instance terminates */ + } + + unlink(NETTEST_INPUT_CONF_FILE); +} + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int fd; + int ret = 0; + FILE *fp; + char buf[4096]; + char testcases[4096]; + + if(argc != 3) { + printf("Invalid call of nettester\n"); + printf("Usage: nettester testsuite-name input\n"); + printf(" input = udp|tcp\n"); + exit(1); + } + + atexit(doAtExit); + + testSuite = argv[1]; + + if(!strcmp(argv[2], "udp")) + inputMode = inputUDP; + else if(!strcmp(argv[2], "tcp")) + inputMode = inputTCP; + else { + printf("error: unsupported input mode '%s'\n", argv[2]); + exit(1); + } + + if((srcdir = getenv("srcdir")) == NULL) + srcdir = "."; + + printf("Start of nettester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + + /* create input config file */ + if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { + perror(NETTEST_INPUT_CONF_FILE); + printf("error opening input configuration file\n"); + exit(1); + } + if(inputMode == inputUDP) { + fputs("$ModLoad ../plugins/imudp/.libs/imudp\n", fp); + fputs("$UDPServerRun 12514\n", fp); + } else { + fputs("$ModLoad ../plugins/imtcp/.libs/imtcp\n", fp); + fputs("$InputTCPServerRun 13514\n", fp); + } + fclose(fp); + + /* start to be tested rsyslogd */ + openPipe(argv[1], &rsyslogdPid, &fd); + readLine(fd, buf); + + /* generate filename */ + sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); + if(doTests(fd, testcases) != 0) + ret = 1; + + printf("End of nettester run (%d).\n", ret); + exit(ret); +} diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index cac08928..8a8d67f3 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1 +1,5 @@ -./udptester omod-if-array +#!/bin/bash -e +echo test omod-if-array via udp +./nettester omod-if-array udp +echo test omod-if-array via tcp +./nettester omod-if-array tcp diff --git a/tests/parsertest.sh b/tests/parsertest.sh index e7985bb0..fabe7e8d 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1 +1,5 @@ -./udptester parse1 +#!/bin/bash -e +echo test parsertest via udp +./nettester parse1 udp +echo test parsertest via tcp +./nettester parse1 tcp diff --git a/tests/testsuites/omod-if-array.conf b/tests/testsuites/omod-if-array.conf index e6c05a52..d88db166 100644 --- a/tests/testsuites/omod-if-array.conf +++ b/tests/testsuites/omod-if-array.conf @@ -3,8 +3,7 @@ # the testbench, so we do not need to focus on that) # rgerhards, 2009-04-03 $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ActionOMStdoutArrayInterface on $ErrorMessagesToStderr off diff --git a/tests/testsuites/parse1.conf b/tests/testsuites/parse1.conf index 0fb7d16d..947a05a8 100644 --- a/tests/testsuites/parse1.conf +++ b/tests/testsuites/parse1.conf @@ -1,6 +1,5 @@ $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ErrorMessagesToStderr off diff --git a/tests/udptester.c b/tests/udptester.c deleted file mode 100644 index a3c6658d..00000000 --- a/tests/udptester.c +++ /dev/null @@ -1,293 +0,0 @@ -/* Runs a test suite on the rsyslog (and later potentially - * other things). - * - * The name of the test suite must be given as argv[1]. In this config, - * rsyslogd is loaded with config ./testsuites/.conf and then - * test cases ./testsuites/ *. are executed on it. This test driver is - * suitable for testing cases where a message sent (via UDP) results in - * exactly one response. It can not be used in cases where no response - * is expected (that would result in a hang of the test driver). - * Note: each test suite can contain many tests, but they all need to work - * with the same rsyslog configuration. - * - * Part of the testbench for rsyslog. - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define EXIT_FAILURE 1 - -static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ -static char *testSuite; /* name of current test suite */ - - -void readLine(int fd, char *ln) -{ - char c; - int lenRead; - lenRead = read(fd, &c, 1); - while(lenRead == 1 && c != '\n') { - *ln++ = c; - lenRead = read(fd, &c, 1); - } - *ln = '\0'; -} - - -/* send a message via UDP - * returns 0 if ok, something else otherwise. - */ -int -udpSend(char *buf, int lenBuf) -{ - struct sockaddr_in si_other; - int s, slen=sizeof(si_other); - - if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { - perror("socket()"); - return(1); - } - - memset((char *) &si_other, 0, sizeof(si_other)); - si_other.sin_family = AF_INET; - si_other.sin_port = htons(12514); - if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { - fprintf(stderr, "inet_aton() failed\n"); - return(1); - } - - if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { - perror("sendto"); - fprintf(stderr, "sendto() failed\n"); - return(1); - } - - close(s); - return 0; -} - - -/* open pipe to test candidate - so far, this is - * always rsyslogd and with a fixed config. Later, we may - * change this. Returns 0 if ok, something else otherwise. - * rgerhards, 2009-03-31 - */ -int openPipe(char *configFile, pid_t *pid, int *pfd) -{ - int pipefd[2]; - pid_t cpid; - char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime//.libs", NULL }; - char confFile[1024]; - char *newenviron[] = { NULL }; - - - sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); - newargv[1] = confFile; - - if (pipe(pipefd) == -1) { - perror("pipe"); - exit(EXIT_FAILURE); - } - - cpid = fork(); - if (cpid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } - - if(cpid == 0) { /* Child reads from pipe */ - fclose(stdout); - dup(pipefd[1]); - close(pipefd[1]); - close(pipefd[0]); - fclose(stdin); - execve("../tools/rsyslogd", newargv, newenviron); - } else { - close(pipefd[1]); - *pid = cpid; - *pfd = pipefd[0]; - } - - return(0); -} - - -/* Process a specific test case. File name is provided. - * Needs to return 0 if all is OK, something else otherwise. - */ -int -processTestFile(int fd, char *pszFileName) -{ - FILE *fp; - char *testdata = NULL; - char *expected = NULL; - int ret = 0; - size_t lenLn; - char buf[4096]; - - if((fp = fopen((char*)pszFileName, "r")) == NULL) { - perror((char*)pszFileName); - return(2); - } - - /* skip comments at start of file */ - - getline(&testdata, &lenLn, fp); - while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } - - - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send */ - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ - - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; - } - - free(testdata); - free(expected); - fclose(fp); - return(ret); -} - - -/* carry out all tests. Tests are specified via a file name - * wildcard. Each of the files is read and the test carried - * out. - * Returns the number of tests that failed. Zero means all - * success. - */ -int -doTests(int fd, char *files) -{ - int iFailed = 0; - int iTests = 0; - int ret; - char *testFile; - glob_t testFiles; - size_t i = 0; - struct stat fileInfo; - - glob(files, GLOB_MARK, NULL, &testFiles); - - for(i = 0; i < testFiles.gl_pathc; i++) { - testFile = testFiles.gl_pathv[i]; - - if(stat((char*) testFile, &fileInfo) != 0) - continue; /* continue with the next file if we can't stat() the file */ - - ++iTests; - /* all regular files are run through the test logic. Symlinks don't work. */ - if(S_ISREG(fileInfo.st_mode)) { /* config file */ - printf("processing test case '%s' ... ", testFile); - ret = processTestFile(fd, testFile); - if(ret == 0) { - printf("successfully completed\n"); - } else { - printf("failed!\n"); - ++iFailed; - } - } - } - globfree(&testFiles); - - if(iTests == 0) { - printf("Error: no test cases found, no tests executed.\n"); - iFailed = 1; - } else { - printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); - } - - return(iFailed); -} - - -/* Run the test suite. This must be called with exactly one parameter, the - * name of the test suite. For details, see file header comment at the top - * of this file. - * rgerhards, 2009-04-03 - */ -int main(int argc, char *argv[]) -{ - int fd; - pid_t pid; - int status; - int ret = 0; - char buf[4096]; - char testcases[4096]; - - if(argc != 2) { - printf("Invalid call of udptester\n"); - printf("Usage: udptester testsuite-name\n"); - exit(1); - } - - testSuite = argv[1]; - - if((srcdir = getenv("srcdir")) == NULL) - srcdir = "."; - - printf("Start of udptester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); - - openPipe(argv[1], &pid, &fd); - readLine(fd, buf); - - /* generate filename */ - sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); - if(doTests(fd, testcases) != 0) - ret = 1; - - /* cleanup */ - kill(pid, SIGTERM); - waitpid(pid, &status, 0); /* wait until instance terminates */ - printf("End of udptester run.\n"); - exit(ret); -} -- cgit From 7cc7166cb24e50e48cab1cbb3c19c8c17d95ace7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 17:48:02 +0200 Subject: added new test case for many tcp connections It is checked that many tcp connections are properly handled. While adding this test, I noticed that there is a bug in imtcp that prevents creation of more than 200 connections. This bug still exists, so the test suite currently fails (what is correct). Will be addressed soon. --- ChangeLog | 4 +- tests/Makefile.am | 7 +- tests/chkseq.c | 76 +++++++++++++++++ tests/manytcp.sh | 13 +++ tests/tcpflood | Bin 0 -> 17940 bytes tests/tcpflood.c | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 tests/chkseq.c create mode 100755 tests/manytcp.sh create mode 100755 tests/tcpflood create mode 100644 tests/tcpflood.c diff --git a/ChangeLog b/ChangeLog index 98146600..98b4f235 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,7 +5,9 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" -- improved testbench, added tests for tcp-based reception +- improved testbench + * added tests for tcp-based reception + * added tcp-load test (1000 connections, 20,000 messages) --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/tests/Makefile.am b/tests/Makefile.am index fa662a3e..53a81a93 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -27,10 +27,13 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ parsertest.sh \ + manytcp.sh \ omod-if-array.sh \ cfg.sh ourtail_SOURCES = ourtail.c +tcpflood_SOURCES = tcpflood.c +chkseq_SOURCES = chkseq.c nettester_SOURCES = nettester.c getline.c nettester_LDADD = $(SOL_LIBS) diff --git a/tests/chkseq.c b/tests/chkseq.c new file mode 100644 index 00000000..3203c250 --- /dev/null +++ b/tests/chkseq.c @@ -0,0 +1,76 @@ +/* Checks if a file consists of line of strictly monotonically + * increasing numbers. An expected start and end number may + * be set. + * + * Params + * argv[1] file to check + * argv[2] start number + * argv[3] end number + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include + +int main(int argc, char *argv[]) +{ + FILE *fp; + int val; + int i; + int ret = 0; + int start, end; + + if(argc != 4) { + printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file start end\n"); + exit(1); + } + + start = atoi(argv[2]); + end = atoi(argv[3]); + + if(start > end) { + printf("start must be less than or equal end!\n"); + exit(1); + } + + /* read file */ + fp = fopen(argv[1], "r"); + if(fp == NULL) { + perror(argv[1]); + exit(1); + } + + for(i = start ; i < end ; ++i) { + if(fscanf(fp, "%d\n", &val) != 1) { + printf("scanf error in index i=%d\n", i); + exit(1); + } + if(val != i) { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } + } + + exit(ret); +} diff --git a/tests/manytcp.sh b/tests/manytcp.sh new file mode 100755 index 00000000..3accfb8a --- /dev/null +++ b/tests/manytcp.sh @@ -0,0 +1,13 @@ +rm -f rsyslog.out.log # work file +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -ftestsuites/manytcp.conf & +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13514 1000 20000 +sleep 1 +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 19999 +if [ "$?" -ne "0" ]; then + echo "sequence error detected" + exit 1 +fi diff --git a/tests/tcpflood b/tests/tcpflood new file mode 100755 index 00000000..90b89b84 Binary files /dev/null and b/tests/tcpflood differ diff --git a/tests/tcpflood.c b/tests/tcpflood.c new file mode 100644 index 00000000..cecec628 --- /dev/null +++ b/tests/tcpflood.c @@ -0,0 +1,245 @@ +/* Opens a large number of tcp connections and sends + * messages over them. This is used for stress-testing. + * + * Params + * argv[1] target address + * argv[2] target port + * argv[3] number of connections + * argv[4] number of messages to send (connection is random) + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static char *targetIP; +static int targetPort; +static int numMsgsToSend; /* number of messages to send */ +static int numConnections; /* number of connections to create */ +static int *sockArray; /* array of sockets to use */ + + +/* open a single tcp connection + */ +int openConn(int *fd) +{ + int sock; + struct sockaddr_in addr; + + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(targetPort); + if(inet_aton(targetIP, &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + perror("connect()"); + fprintf(stderr, "connect() failed\n"); + return(1); + } + + *fd = sock; + return 0; +} + + +/* open all requested tcp connections + * this includes allocating the connection array + */ +int openConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " open connections", sizeof(" open connections")-1); + sockArray = calloc(numConnections, sizeof(int)); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); + write(1, msgBuf, lenMsg); + } + if(openConn(&(sockArray[i])) != 0) { + printf("error in trying to open connection i=%d\n", i); + return 1; + } + } + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d open connections\n", i); + write(1, msgBuf, lenMsg); + + return 0; +} + + +/* send messages to the tcp connections we keep open. We use + * a very basic format that helps identify the message + * (via msgnum:: e.g. msgnum:00000001:). This format is suitable + * for extracton to field-based properties. + * The first numConnection messages are sent sequentially, as are the + * last. All messages in between are sent over random connections. + * Note that message numbers start at 0. + */ +int sendMessages(void) +{ + int i; + int socknum; + int lenBuf; + char buf[2048]; + char msgBuf[128]; + size_t lenMsg; + + srand(time(NULL)); /* seed is good enough for our needs */ + + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent", 0); + write(1, msgBuf, lenMsg); + for(i = 0 ; i < numMsgsToSend ; ++i) { + if(i < numConnections) + socknum = i; + else if(i >= numMsgsToSend - numConnections) + socknum = i - (numMsgsToSend - numConnections); + else + socknum = rand() % numConnections; + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + if(send(sockArray[socknum], buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + if(i % 100 == 0) { + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); + write(1, msgBuf, lenMsg); + } + } + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent\n", i); + write(1, msgBuf, lenMsg); + + return 0; +} + + +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int ret = 0; + static char buf[1024]; + + setvbuf(stdout, _IONBF, buf, 48); + + if(argc != 5) { + printf("Invalid call of tcpflood\n"); + printf("Usage: nettester testsuite-name input\n"); + exit(1); + } + + targetIP = argv[1]; + targetPort = atoi(argv[2]); + numConnections = atoi(argv[3]); + numMsgsToSend = atoi(argv[4]); + + if(openConnections() != 0) { + printf("error opening connections\n"); + exit(1); + } + + if(sendMessages() != 0) { + printf("error sending messages\n"); + exit(1); + } + exit(ret); +} -- cgit From de38f744de1ce746e6e61098fd4a05ac76ef71a0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 18:18:31 +0200 Subject: minor cleanup --- tests/tcpflood | Bin 17940 -> 17972 bytes tests/tcpflood.c | 2 +- tools/syslogd.c | 8 -------- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/tcpflood b/tests/tcpflood index 90b89b84..ae00fcd5 100755 Binary files a/tests/tcpflood and b/tests/tcpflood differ diff --git a/tests/tcpflood.c b/tests/tcpflood.c index cecec628..83f0d1ee 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -223,7 +223,7 @@ int main(int argc, char *argv[]) if(argc != 5) { printf("Invalid call of tcpflood\n"); - printf("Usage: nettester testsuite-name input\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); exit(1); } diff --git a/tools/syslogd.c b/tools/syslogd.c index 0badac19..a4f0059b 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3591,18 +3591,10 @@ int realMain(int argc, char **argv) fprintf(stderr, "-t option only supported in compatibility modes 0 to 2 - ignored\n"); break; case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ -{ -char buf[1024]; -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -printf("chroot to '%s'\n", arg); if(chroot(arg) != 0) { perror("chroot"); exit(1); } -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -} break; case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); -- cgit From 1cfa08749b1c474de850f693915b9a32d456c593 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 18:59:15 +0200 Subject: bugfix: solved potential memory leak in msg processing could manifest itself in imtcp (loss of a few bytes for *each* received message - but depended on config) -- this was newly introduced --- ChangeLog | 2 ++ runtime/msg.c | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 98146600..c99cc00d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,8 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? of loadable library modules. - added new RainerScript function "tolower" - improved testbench, added tests for tcp-based reception +- bugfix: solved potential memory leak in msg processing, could manifest + itself in imtcp --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/runtime/msg.c b/runtime/msg.c index 5d1f21fd..9d5f3838 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1542,6 +1542,8 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) { assert(pMsg != NULL); assert(pBuf != NULL); + if(pMsg->pszHOSTNAME != NULL) + free(pMsg->pszHOSTNAME); pMsg->iLenHOSTNAME = strlen(pBuf); pMsg->pszHOSTNAME = (uchar*) pBuf; } @@ -1567,7 +1569,7 @@ void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME) if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); else - dbgprintf("Could not allocate memory in MsgSetHOSTNAME()\n"); + DBGPRINTF("Could not allocate memory in MsgSetHOSTNAME()\n"); } -- cgit From d7f11ecb06688186d4c68b5933fb1437279ce03d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 8 Apr 2009 16:50:57 +0200 Subject: Make it work in batches of statements. Currently, all statements to be executed are stored on the same structure. When the batch size is reached, all statements are executed in a single transaction, and then committed. There are many corner cases in which an error may happen and the batch may be left in an inconsistent state, perhaps leaking memory or crashing. They will be fixed. --- plugins/omoracle/omoracle.c | 71 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ea910d3a..9b1ec42d 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -15,7 +15,8 @@ $OmoracleDBPassword: password to log in on the database. $OmoracleDB: connection string (an Oracle easy connect or a db name as specified by tnsnames.ora) - + $OmoracleBatchSize: Number of elements to send to the DB. + All fields are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -50,6 +51,18 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +/** */ +struct oracle_batch +{ + /* Batch size */ + int size; + /* Last element inserted in the buffer. The batch will be + * executed when n == size */ + int n; + /* Statements to run on this transaction */ + char** statements; +}; + typedef struct _instanceData { /* Environment handler, the base for any OCI work. */ OCIEnv* environment; @@ -67,6 +80,8 @@ typedef struct _instanceData { OCIBind* binding; /* Connection string, kept here for possible retries. */ char* connection; + /* Batch */ + struct oracle_batch batch; } instanceData; /** Database name, to be filled by the $OmoracleDB directive */ @@ -76,6 +91,8 @@ static char* db_name; static char* db_user; /** Database password, to be filled by the $OmoracleDBPassword */ static char* db_password; +/** Batch size. */ +static int batch_size; /** Generic function for handling errors from OCI. @@ -148,6 +165,12 @@ CODESTARTcreateInstance OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); + pData->batch.n = 0; + pData->batch.size = batch_size; + pData->batch.statements = calloc(pData->batch.size, + sizeof *pData->batch.statements); + CHKmalloc(pData->batch.statements); + finalize_it: ENDcreateInstance @@ -163,6 +186,9 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); + while (pData->batch.size--) + free(pData->batch.statements[pData->batch.size]); + free(pData->batch.statements); dbgprintf ("omoracle freed all its resources\n"); RETiRet; @@ -263,21 +289,41 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction + int i; + int n; CODESTARTdoAction dbgprintf("omoracle attempting to execute statement %s\n", *ppString); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, pData->error, *ppString, - strlen(*ppString), OCI_NTV_SYNTAX, - OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, pData->statement, pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); + + if (pData->batch.n == pData->batch.size) { + dbgprintf("omoracle batch size limit hit, sending into DB\n"); + for (i = 0; i < pData->batch.n; i++) { + if (pData->batch.statements[i] == NULL) + continue; + n = strlen(pData->batch.statements[i]); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->batch.statements[i], n, + OCI_NTV_SYNTAX, OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + free(pData->batch.statements[i]); + pData->batch.statements[i] = NULL; + } + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); + pData->batch.n = 0; + } + pData->batch.statements[pData->batch.n] = strdup(*ppString); + CHKmalloc(pData->batch.statements[pData->batch.n]); + pData->batch.n++; + finalize_it: dbgprintf ("omoracle %s at executing statement %s\n", iRet?"did not succeed":"succeeded", *ppString); -/* Clean credentials to avoid leakage in case of core dump. */ ENDdoAction BEGINmodExit @@ -331,4 +377,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, eCmdHdlrGetWord, NULL, &db_name, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchsize", 0, + eCmdHdlrInt, NULL, &batch_size, + STD_LOADABLE_MODULE_ID)); ENDmodInit -- cgit From 9bbd5dfd25f3e2a7a6839ff0e7b4076186efbcf4 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 8 Apr 2009 16:50:58 +0200 Subject: Solve a memory leak when freeing Oracle instances. --- plugins/omoracle/omoracle.c | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 9b1ec42d..df06a8b6 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -190,7 +190,6 @@ CODESTARTfreeInstance free(pData->batch.statements[pData->batch.size]); free(pData->batch.statements); dbgprintf ("omoracle freed all its resources\n"); - RETiRet; ENDfreeInstance -- cgit From 1ebbd3a2df09ed9e706be7762307cd17c4a5cad0 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 8 Apr 2009 16:50:59 +0200 Subject: Stop omoracle losing messages on rsyslog shutdown. When rsyslog shuts down, we must send and commit any pending messages or information will be lost. It will make rsyslog's shut down slower, but also more reliable. --- plugins/omoracle/omoracle.c | 65 +++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index df06a8b6..f6679953 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -12,13 +12,17 @@ namely: $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db name as specified by tnsnames.ora) - $OmoracleBatchSize: Number of elements to send to the DB. - All fields are mandatory. The dbstring can be an Oracle easystring - or a DB name, as present in the tnsnames.ora file. + $OmoracleBatchSize: Number of elements to send to the DB on each + transaction. + + All these directives are mandatory. The dbstring can be an Oracle + easystring or a DB name, as present in the tnsnames.ora file. Author: Luis Fernando Muñoz Mejías @@ -174,11 +178,45 @@ CODESTARTcreateInstance finalize_it: ENDcreateInstance +/* Inserts all stored statements into the database, releasing any + * allocated memory. */ +static int insert_to_db(instanceData* pData) +{ + DEFiRet; + int i, n; + + for (i = 0; i < pData->batch.n; i++) { + if (pData->batch.statements[i] == NULL) + continue; + n = strlen(pData->batch.statements[i]); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->batch.statements[i], n, + OCI_NTV_SYNTAX, OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + free(pData->batch.statements[i]); + pData->batch.statements[i] = NULL; + } + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); + pData->batch.n = 0; +finalize_it: + RETiRet; +} + /** Close the session and free anything allocated by createInstance. */ BEGINfreeInstance CODESTARTfreeInstance +/* Before actually releasing our resources, let's try to commit + * anything pending so that we don't lose any messages. */ + insert_to_db(pData); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->environment, OCI_HTYPE_ENV); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); @@ -295,26 +333,7 @@ CODESTARTdoAction if (pData->batch.n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); - for (i = 0; i < pData->batch.n; i++) { - if (pData->batch.statements[i] == NULL) - continue; - n = strlen(pData->batch.statements[i]); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, - pData->error, - pData->batch.statements[i], n, - OCI_NTV_SYNTAX, OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, - pData->statement, - pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - free(pData->batch.statements[i]); - pData->batch.statements[i] = NULL; - } - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); - pData->batch.n = 0; + CHKiRet(insert_to_db(pData)); } pData->batch.statements[pData->batch.n] = strdup(*ppString); CHKmalloc(pData->batch.statements[pData->batch.n]); -- cgit From cf0cf2fa4bf84188baf873522d4a22887f4fdf90 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 13:57:35 +0200 Subject: removed accidently added binary --- tests/tcpflood | Bin 17972 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 tests/tcpflood diff --git a/tests/tcpflood b/tests/tcpflood deleted file mode 100755 index ae00fcd5..00000000 Binary files a/tests/tcpflood and /dev/null differ -- cgit From 1e52abd8be2dfde8727c7e54b02a10ce5051815f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 14:14:00 +0200 Subject: fixing "make distcheck" --- tests/testsuites/manytcp.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/testsuites/manytcp.conf diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf new file mode 100644 index 00000000..e491cd04 --- /dev/null +++ b/tests/testsuites/manytcp.conf @@ -0,0 +1,11 @@ +# Test for tcp "flood" testing +# rgerhards, 2009-04-08 +$ModLoad ../plugins/imtcp/.libs/imtcp +$inputtcpmaxsessions 2000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit From d2d8cc9c1b65bedeafaa3a06ed53cbc8511be9a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 14:45:23 +0200 Subject: fixed testbench compilation problem on Solaris Solaris network libraries needed to be specified in linker options --- tests/Makefile.am | 4 +++- tests/manytcp.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 6d3c8725..87dca985 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -33,9 +33,11 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg.sh ourtail_SOURCES = ourtail.c -tcpflood_SOURCES = tcpflood.c chkseq_SOURCES = chkseq.c +tcpflood_SOURCES = tcpflood.c +tcpflood_LDADD = $(SOL_LIBS) + nettester_SOURCES = nettester.c getline.c nettester_LDADD = $(SOL_LIBS) diff --git a/tests/manytcp.sh b/tests/manytcp.sh index b38245df..f0a3eb96 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,5 +1,6 @@ rm -f rsyslog.out.log # work file ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & +sleep 1 echo "rsyslogd started with pid " `cat rsyslog.pid` ./tcpflood 127.0.0.1 13514 1000 20000 sleep 1 -- cgit From e07b3f380f87166ce055ac4f3253f0367ef5cdce Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 18:51:21 +0200 Subject: working some more on "make distcheck" ... this time I think successfully (at least on Fedora...) --- Makefile.am | 2 +- tests/Makefile.am | 1 + tests/manytcp.sh | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index dfb33339..58e1cfce 100644 --- a/Makefile.am +++ b/Makefile.am @@ -127,5 +127,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-oracle +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout ACLOCAL_AMFLAGS = -I m4 diff --git a/tests/Makefile.am b/tests/Makefile.am index 53a81a93..6d3c8725 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -28,6 +28,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.omod-if-array \ parsertest.sh \ manytcp.sh \ + testsuites/manytcp.conf \ omod-if-array.sh \ cfg.sh diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 3accfb8a..b38245df 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,5 +1,5 @@ rm -f rsyslog.out.log # work file -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -ftestsuites/manytcp.conf & +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & echo "rsyslogd started with pid " `cat rsyslog.pid` ./tcpflood 127.0.0.1 13514 1000 20000 sleep 1 @@ -8,6 +8,8 @@ rm -f work sort < rsyslog.out.log > work ./chkseq work 0 19999 if [ "$?" -ne "0" ]; then + rm -f work rsyslog.out.log echo "sequence error detected" exit 1 fi +rm -f work rsyslog.out.log -- cgit From 9633ce1afcb3c3e44f969be0503360a4c0c75599 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 19:17:35 +0200 Subject: fixed compile-time problems in im3195 ... however, I did not not a test run due to the lack of existing test drivers and the very low (aka "non-existing" interest from the userbase in the feature). --- plugins/im3195/im3195.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/im3195/im3195.c b/plugins/im3195/im3195.c index 1c2502fe..106da2c8 100644 --- a/plugins/im3195/im3195.c +++ b/plugins/im3195/im3195.c @@ -47,6 +47,7 @@ #include "liblogging/syslogmessage.h" #include "module-template.h" #include "cfsysline.h" +#include "msg.h" #include "errmsg.h" MODULE_TYPE_INPUT @@ -83,7 +84,7 @@ void OnReceive(srAPIObj __attribute__((unused)) *pMyAPI, srSLMGObj* pSLMG) srSLMGGetRawMSG(pSLMG, &pszRawMsg); parseAndSubmitMessage(fromHost, fromHostIP, pszRawMsg, strlen((char*)pszRawMsg), - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_FULL_DELAY, (uchar*)"im3195"); + PARSE_HOSTNAME, eFLOWCTL_FULL_DELAY, (uchar*)"im3195", NULL, 0); } -- cgit From b9b96fbfc66809532f4ed24d667947fa63ce68f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 19:38:17 +0200 Subject: removed MSG_NOSIGNAL & provided work-around as this send() option is not supported on Solaris. We now simply ignore SIGPIPE --- tests/tcpflood.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 254e9fd6..fcb68998 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -170,7 +170,7 @@ int sendMessages(void) else socknum = rand() % numConnections; lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); - lenSend = send(sockArray[socknum], buf, lenBuf, MSG_NOSIGNAL); + lenSend = send(sockArray[socknum], buf, lenBuf, 0); if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); @@ -250,8 +250,17 @@ tcpSend(char *buf, int lenBuf) int main(int argc, char *argv[]) { int ret = 0; + struct sigaction sigAct; static char buf[1024]; + /* on Solaris, we do not HAVE MSG_NOSIGNAL, so for this reason + * we block SIGPIPE (not an issue for this program) + */ + memset(&sigAct, 0, sizeof(sigAct)); + sigemptyset(&sigAct.sa_mask); + sigAct.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sigAct, NULL); + setvbuf(stdout, buf, _IONBF, 48); if(argc != 5) { -- cgit From 8d65a9cdd49b812fb86cfd1b33f01c27e995db36 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 14:55:37 +0200 Subject: updated project status --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index dae94884..fff3a6ff 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,15 +5,15 @@

    This page reflects the status as of 2009-04-03.

    Current Releases

    -

    development: 4.1.5 [2009-03-11] - -change log - -download +

    development: 4.1.6 [2009-04-07] - +change log - +download
    beta: 3.21.11 [2009-04-03] - change log - download

    -

    v3 stable: 3.20.3 [2009-04-02] - change log - +

    v3 stable: 3.20.5 [2009-04-02] - change log - download
    v2 stable: 2.0.6 [2008-08-07] - change log - -- cgit From 04272876d12488b2039b28683dc53e1c802d303d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 13:52:07 +0200 Subject: implemented $MaxOpenFiles directive and changed testbench ... to utilize it. This work is not yet fully verified to be correct. --- runtime/rsyslog.h | 1 + tests/manytcp.sh | 13 +++++++--- tests/tcpflood.c | 59 +++++++++++++++++++++++++++++++++++-------- tests/testsuites/manytcp.conf | 4 ++- tools/syslogd.c | 28 ++++++++++++++++++++ 5 files changed, 89 insertions(+), 16 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index cea457d8..25ec30fc 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -262,6 +262,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ + RS_RET_ERR_RLIM_NOFILE = -2116, /**< error setting max. nbr open files process limit */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/tests/manytcp.sh b/tests/manytcp.sh index f0a3eb96..d9b2e9a0 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,13 +1,18 @@ -rm -f rsyslog.out.log # work file +rm -f work rsyslog.out.log rsyslog.out.log.save # work files ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & sleep 1 echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13514 1000 20000 -sleep 1 +# the config file specifies exactly 1100 connections +./tcpflood 127.0.0.1 13514 1000 40000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 5 # we need this so that rsyslogd can receive all outstanding messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 19999 +./chkseq work 0 39999 if [ "$?" -ne "0" ]; then rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 83f0d1ee..254e9fd6 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -95,21 +95,49 @@ int openConnections(void) sockArray = calloc(numConnections, sizeof(int)); for(i = 0 ; i < numConnections ; ++i) { if(i % 10 == 0) { - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d", i); + //lenMsg = sprintf(msgBuf, "\r%5.5d", i); + //write(1, msgBuf, lenMsg); } if(openConn(&(sockArray[i])) != 0) { printf("error in trying to open connection i=%d\n", i); return 1; } } - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d open connections\n", i); + lenMsg = sprintf(msgBuf, "\r%5.5d open connections\n", i); write(1, msgBuf, lenMsg); return 0; } +/* we also close all connections because otherwise we may get very bad + * timing for the syslogd - it may not be able to process all incoming + * messages fast enough if we immediately shut down. + * TODO: it may be an interesting excercise to handle that situation + * at the syslogd level, too + * rgerhards, 2009-04-14 + */ +void closeConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " close connections", sizeof(" close connections")-1); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + lenMsg = sprintf(msgBuf, "\r%5.5d", i); + write(1, msgBuf, lenMsg); + } + close(sockArray[i]); + } + lenMsg = sprintf(msgBuf, "\r%5.5d close connections\n", i); + write(1, msgBuf, lenMsg); + +} + + /* send messages to the tcp connections we keep open. We use * a very basic format that helps identify the message * (via msgnum:: e.g. msgnum:00000001:). This format is suitable @@ -123,13 +151,16 @@ int sendMessages(void) int i; int socknum; int lenBuf; + int lenSend; char buf[2048]; char msgBuf[128]; size_t lenMsg; srand(time(NULL)); /* seed is good enough for our needs */ - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent", 0); + printf("Sending %d messages.\n", numMsgsToSend); + printf("\r%5.5d messages sent", 0); + lenMsg = sprintf(msgBuf, "\r%5.5d/%5.5d messages sent", 0, numMsgsToSend); write(1, msgBuf, lenMsg); for(i = 0 ; i < numMsgsToSend ; ++i) { if(i < numConnections) @@ -139,18 +170,20 @@ int sendMessages(void) else socknum = rand() % numConnections; lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); - if(send(sockArray[socknum], buf, lenBuf, 0) != lenBuf) { + lenSend = send(sockArray[socknum], buf, lenBuf, MSG_NOSIGNAL); + if(lenSend != lenBuf) { + printf("\r%5.5d\n", i); + fflush(stdout); perror("send test data"); - fprintf(stderr, "send() failed\n"); + printf("send() failed at socket %d, index %d\n", socknum, i); + fflush(stderr); return(1); } if(i % 100 == 0) { - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d", i); } } - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent\n", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d messages sent\n", i); return 0; } @@ -219,7 +252,7 @@ int main(int argc, char *argv[]) int ret = 0; static char buf[1024]; - setvbuf(stdout, _IONBF, buf, 48); + setvbuf(stdout, buf, _IONBF, 48); if(argc != 5) { printf("Invalid call of tcpflood\n"); @@ -241,5 +274,9 @@ int main(int argc, char *argv[]) printf("error sending messages\n"); exit(1); } + + //closeConnections(); + printf("End of tcpflood Run\n"); + exit(ret); } diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index e491cd04..8175732e 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,7 +1,9 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 $ModLoad ../plugins/imtcp/.libs/imtcp -$inputtcpmaxsessions 2000 +$MainMsgQueueTimeoutShutdown 10000 +$MaxOpenFiles 2000 +$InputTCPMaxSessions 1100 $InputTCPServerRun 13514 $ErrorMessagesToStderr off diff --git a/tools/syslogd.c b/tools/syslogd.c index a4f0059b..8c86c12e 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -87,6 +87,7 @@ #include #include #include +#include #include #if HAVE_SYS_TIMESPEC_H @@ -2073,6 +2074,32 @@ static rsRetVal setActionResumeInterval(void __attribute__((unused)) *pVal, int } +/* set the processes max number ob files (upon configuration request) + * 2009-04-14 rgerhards + */ +static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) +{ + struct rlimit maxFiles; + char errStr[1024]; + DEFiRet; + + maxFiles.rlim_cur = iFiles; + maxFiles.rlim_max = iFiles; + + if(setrlimit(RLIMIT_NOFILE, &maxFiles) < 0) { + /* NOTE: under valgrind, we seem to be unable to extend the size! */ + rs_strerror_r(errno, errStr, sizeof(errStr)); + errmsg.LogError(0, RS_RET_ERR_RLIM_NOFILE, "could not set process file limit to %d: %s [kernel max %ld]", + iFiles, errStr, (long) maxFiles.rlim_max); + ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); + } + dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); + +finalize_it: + RETiRet; +} + + /* set the processes umask (upon configuration request) */ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { @@ -2870,6 +2897,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"modload", 0, eCmdHdlrCustomHandler, conf.doModLoad, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"includeconfig", 0, eCmdHdlrCustomHandler, conf.doIncludeLine, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"umask", 0, eCmdHdlrFileCreateMode, setUmask, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"maxopenfiles", 0, eCmdHdlrInt, setMaxFiles, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprinttemplatelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintTemplateList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintmodulelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintModuleList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintcfsyslinehandlerlist", 0, eCmdHdlrBinary, -- cgit From 01e5d51c57536acfa8e05d5b22c001a6cf0701d8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 14:36:50 +0200 Subject: added doc for $MaxOpenFiles directive --- ChangeLog | 1 + doc/rsconf1_maxopenfiles.html | 35 +++++++++++++++++++++++++++++++++++ doc/rsyslog_conf.html | 6 +++--- doc/rsyslog_conf_global.html | 18 +++++++++--------- 4 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 doc/rsconf1_maxopenfiles.html diff --git a/ChangeLog b/ChangeLog index 792c108c..3108cdb3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? - improved testbench * added tests for tcp-based reception * added tcp-load test (1000 connections, 20,000 messages) +- added $MaxOpenFiles configuration directive - bugfix: solved potential memory leak in msg processing, could manifest itself in imtcp --------------------------------------------------------------------------- diff --git a/doc/rsconf1_maxopenfiles.html b/doc/rsconf1_maxopenfiles.html new file mode 100644 index 00000000..b6c9cc0e --- /dev/null +++ b/doc/rsconf1_maxopenfiles.html @@ -0,0 +1,35 @@ + + +$MaxOpenFiles - rsyslog.conf file + + +[rsyslog configuration directive overview] + +

    $MaxOpenFiles

    +

    Available Since: 4.3.0

    +

    Type: global configuration directive

    +

    Default: operating system default

    +

    Description:

    +

    Set the maximum number of files that the rsyslog process can have open at any given +time. Note that this includes open tcp sockets, so this setting is the upper limit for +the number of open TCP connections as well. If you expect a large nubmer of concurrent +connections, it is suggested that the number is set to the max number connected plus 1000. +Please note that each dynafile also requires up to 100 open file handles. +

    The setting is similar to running "ulimit -n number-of-files". +

    Please note that depending on permissions and operating system configuration, the +setrlimit() request issued by rsyslog may fail, in which case the previous limit is kept +in effect. Rsyslog will emit a warning message in this case. +

    Sample:

    +

    $MaxOpenFiles 2000

    +

    Bugs:

    +

    For some reason, this settings seems not to work on all platforms. If you experience +problems, please let us know so that we can (hopefully) narrow down the issue. +

    [rsyslog.conf overview] [manual +index] [rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 3 or higher.

    + + diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 852d95b5..6990c6bd 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -26,7 +26,7 @@ Lines can be continued by specifying a backslash ("\") as the last character of the line. There is a hard-coded maximum line length of 4K. If you need lines larger than that, you need to change compile-time settings inside rsyslog and recompile. -

    Global Directives

    +

    Configuration Directives

    Basic Structure

    Rsyslog supports standard sysklogd's configuration file format and extends it. So in general, you can take a "normal" syslog.conf and @@ -74,9 +74,9 @@ such features is available in rsyslogd, only.

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008,2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    > diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d011bd2b..3e33f0da 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -1,14 +1,14 @@ -Global Directives - rsyslog.conf +Configuration Directives - rsyslog.conf

    This is a part of the rsyslog.conf documentation.

    back -

    Global Directives

    -

    All global directives need to be specified on a line by their -own and must start with a dollar-sign. Here is a list in alphabetical -order. Follow links for a description.

    -

    Please note that not all directives here are actually global. Some affect -only the next action. This documentation will be changed soon. +

    Configuration Directives

    +

    All configuration directives need to be specified on a line by their +own and must start with a dollar-sign. Note that those starting with +the word "Action" modify the next action and should be specified +in front of it. +

    Here is a list in alphabetical order. Follow links for a description.

    Not all directives have an in-depth description right now. Default values for them are in bold. A more in-depth description will appear as implementation progresses. @@ -180,6 +180,7 @@ instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restri
    Note that 2k, the current default, is the smallest size that must be supported in order to be compliant to the upcoming new syslog RFC series.

    • +
  • $MaxOpenFiles
  • $ModDir
  • $ModLoad
  • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, @@ -214,7 +215,6 @@ the value, the less precise the timestamp.
  • $PrivDropToGroupID
  • $PrivDropToUser
  • $PrivDropToUserID
    • -
  • $UMASK

    • Where <size_nbr> is specified above, @@ -235,7 +235,7 @@ point of view, "1,,0.0.,.,0" also has the value 1000.

    rsyslog project.
    Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    -- cgit From ba5a59128f6559d58cdd4defe46a9db564d3e2c1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 20:48:08 +0200 Subject: cosmetic fix (status message) --- tests/tcpflood.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index fcb68998..9c17fd5b 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -160,8 +160,6 @@ int sendMessages(void) printf("Sending %d messages.\n", numMsgsToSend); printf("\r%5.5d messages sent", 0); - lenMsg = sprintf(msgBuf, "\r%5.5d/%5.5d messages sent", 0, numMsgsToSend); - write(1, msgBuf, lenMsg); for(i = 0 ; i < numMsgsToSend ; ++i) { if(i < numConnections) socknum = i; -- cgit From 65a85de3d97ab6bc427ea005b75e4b416013de3c Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:16 +0200 Subject: Convert to the array-based interface. We'll receive a single statement to be prepared and a batch size. Then, doAction will execute the statement only once per batch hit, making the process much more efficient. This will reduce network and DB server overhead. The downside is that this version cannot be used with rsyslog v3 anymore. If anyone is interested on backporting the module, they should choose all patches up to this one. Better documentation may follow. --- plugins/omoracle/omoracle.c | 203 +++++++++++++++++++++++++++++++++++--------- plugins/omoracle/omoracle.h | 2 + 2 files changed, 164 insertions(+), 41 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index f6679953..02f83551 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -21,9 +21,28 @@ $OmoracleBatchSize: Number of elements to send to the DB on each transaction. + $OmoracleStatement: Statement to be prepared and executed in + batches. Please note that Oracle's prepared statements have their + placeholders as ':identifier', and this module uses the colon to + guess how many placeholders there will be. + All these directives are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. + The form of the template is just a list of strings you want + inserted to the DB, for instance: + + $template TestStmt,"%hostname%%msg%" + + Will provide the arguments to a statement like + + $OmoracleStatement \ + insert into foo(hostname,message)values(:host,:message) + + Also note that identifiers to placeholders are arbitrarry. You + need to define the properties on the template in the correct order + you want them passed to the statement! + Author: Luis Fernando Muñoz Mejías @@ -40,6 +59,7 @@ #include #include #include +#include #include "dirty.h" #include "syslogd-types.h" #include "srUtils.h" @@ -63,8 +83,12 @@ struct oracle_batch /* Last element inserted in the buffer. The batch will be * executed when n == size */ int n; - /* Statements to run on this transaction */ - char** statements; + /* Number of arguments the statement takes */ + int arguments; + /* Parameters to pass to the statement on this transaction */ + char*** parameters; + /* Binding parameters */ + OCIBind** bindings; }; typedef struct _instanceData { @@ -80,10 +104,10 @@ typedef struct _instanceData { OCISvcCtx* service; /* Credentials object for the connection. */ OCIAuthInfo* authinfo; - /* Binding parameters, currently unused */ - OCIBind* binding; /* Connection string, kept here for possible retries. */ char* connection; + /* Statement to be prepared. */ + char* txt_statement; /* Batch */ struct oracle_batch batch; } instanceData; @@ -97,6 +121,12 @@ static char* db_user; static char* db_password; /** Batch size. */ static int batch_size; +/** Statement to prepare and execute */ +static char* db_statement; +/** Whether or not the core supports the newer array interface. The + * module is able to work in both modes, but the newer is the + * recommended one for performance reasons. */ +static int array_passing; /** Generic function for handling errors from OCI. @@ -149,9 +179,49 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_ERROR; } +/** Returns the number of bind parameters for the statement given as + * an argument. */ +static int count_bind_parameters(char* p) +{ + int n = 0; + + for (; *p; p++) + if (*p == BIND_MARK) + n++; + dbgprintf ("omoracle statement has %d parameters\n", n); + return n; +} + +/** Prepares the statement, binding all its positional parameters */ +static int prepare_statement(instanceData* pData) +{ + int i; + DEFiRet; + + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->txt_statement, + strlen(pData->txt_statement), + OCI_NTV_SYNTAX, OCI_DEFAULT)); + for (i = 0; i < pData->batch.arguments; i++) + CHECKERR(pData->error, + OCIBindByPos(pData->statement, + pData->batch.bindings+i, + pData->error, + i+1, + pData->batch.parameters[i], + sizeof (OCILobLocator*), + SQLT_STR, NULL, NULL, NULL, + 0, 0, OCI_DEFAULT)); +finalize_it: + RETiRet; +} + /* Resource allocation */ BEGINcreateInstance + int i; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -168,12 +238,29 @@ CODESTARTcreateInstance CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); + pData->txt_statement = strdup(db_statement); + CHKmalloc(pData->txt_statement); + dbgprintf("omoracle will run stored statement: %s\n", + pData->txt_statement); pData->batch.n = 0; pData->batch.size = batch_size; - pData->batch.statements = calloc(pData->batch.size, - sizeof *pData->batch.statements); - CHKmalloc(pData->batch.statements); + pData->batch.arguments = count_bind_parameters(pData->txt_statement); + + /* I know, this can be done with a single malloc() call but this is + * easier to read. :) */ + pData->batch.parameters = malloc(pData->batch.arguments * + sizeof *pData->batch.parameters); + CHKmalloc(pData->batch.parameters); + for (i = 0; i < pData->batch.arguments; i++) { + pData->batch.parameters[i] = calloc(pData->batch.size, + sizeof **pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i]); + } + + pData->batch.bindings = calloc(pData->batch.arguments, + sizeof *pData->batch.bindings); + CHKmalloc(pData->batch.bindings); finalize_it: ENDcreateInstance @@ -183,35 +270,34 @@ ENDcreateInstance static int insert_to_db(instanceData* pData) { DEFiRet; - int i, n; + int i, j, n; + + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + pData->batch.n, 0, NULL, NULL, OCI_DEFAULT)); - for (i = 0; i < pData->batch.n; i++) { - if (pData->batch.statements[i] == NULL) - continue; - n = strlen(pData->batch.statements[i]); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, - pData->error, - pData->batch.statements[i], n, - OCI_NTV_SYNTAX, OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, - pData->statement, - pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - free(pData->batch.statements[i]); - pData->batch.statements[i] = NULL; - } CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); + + for (i = 0; i < pData->batch.arguments; i++) + for (j = 0; j < pData->batch.n; j++) { + free(pData->batch.parameters[i][j]); + pData->batch.parameters[i][j] = NULL; + } + pData->batch.n = 0; finalize_it: + dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? + "succeeded" : "did not succeed"); RETiRet; } /** Close the session and free anything allocated by createInstance. */ BEGINfreeInstance + int i, j; CODESTARTfreeInstance /* Before actually releasing our resources, let's try to commit @@ -224,9 +310,13 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); - while (pData->batch.size--) - free(pData->batch.statements[pData->batch.size]); - free(pData->batch.statements); + for (i = 0; i < pData->batch.arguments; i++) { + for (j = 0; j < pData->batch.size; j++) + free(pData->batch.parameters[i][j]); + free(pData->batch.parameters[i]); + } + free(pData->batch.parameters); + free(pData->batch.bindings); dbgprintf ("omoracle freed all its resources\n"); ENDfreeInstance @@ -253,7 +343,7 @@ CODESTARTtryResume * ... of course I don't know why Oracle might need a full restart... * rgerhards, 2009-03-26 */ - dbgprintf("Attempting to reconnect to DB server\n"); + dbgprintf("omoracle attempting to reconnect to DB server\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, @@ -261,6 +351,7 @@ CODESTARTtryResume pData->connection, strlen(pData->connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); + CHKiRet(prepare_statement(pData)); finalize_it: ENDtryResume @@ -296,7 +387,6 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature BEGINparseSelectorAct - CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -314,11 +404,12 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); } CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, - OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + OMSR_TPL_AS_ARRAY, " StdFmt")); CHKiRet(createInstance(&pData)); CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); - + CHKiRet(prepare_statement(pData)); + dbgprintf ("omoracle module got all its resources allocated " "and connected to the DB\n"); @@ -327,21 +418,23 @@ ENDparseSelectorAct BEGINdoAction int i; - int n; + int n = pData->batch.n; + char **params = (char**) ppString[0]; CODESTARTdoAction - dbgprintf("omoracle attempting to execute statement %s\n", *ppString); - if (pData->batch.n == pData->batch.size) { + if (n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); CHKiRet(insert_to_db(pData)); } - pData->batch.statements[pData->batch.n] = strdup(*ppString); - CHKmalloc(pData->batch.statements[pData->batch.n]); + + for (i = 0; i < pData->batch.arguments && params[i]; i++) { + dbgprintf ("omoracle argument on batch[%d][%d]: %s\n", i, n, params[i]); + pData->batch.parameters[i][n] = strdup(params[i]); + CHKmalloc(pData->batch.parameters[i][n]); + } pData->batch.n++; finalize_it: - dbgprintf ("omoracle %s at executing statement %s\n", - iRet?"did not succeed":"succeeded", *ppString); ENDdoAction BEGINmodExit @@ -373,16 +466,35 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, memset(db_password, 0, n); free(db_password); } - db_name = db_user = db_password = NULL; + if (db_statement != NULL) + free(db_statement); + db_name = db_user = db_password = db_statement = NULL; + RETiRet; +} + +/** As I don't find any handler that reads an entire line, I write my + * own. */ +static int get_db_statement(char** line, char** stmt) +{ + DEFiRet; + + while (isspace(**line)) + (*line)++; + dbgprintf ("Config line: %s\n", *line); + *stmt = strdup(*line); + CHKmalloc(*stmt); + dbgprintf ("Statement: %s\n", *stmt); +finalize_it: RETiRet; } BEGINmodInit() + rsRetVal (*supported_options)(unsigned long *pOpts); + unsigned long opts; CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); - /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); @@ -398,4 +510,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchsize", 0, eCmdHdlrInt, NULL, &batch_size, STD_LOADABLE_MODULE_ID)); + CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); + CHKiRet((*supported_options)(&opts)); + if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) + ABORT_FINALIZE(RS_RET_ERR); + + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, + eCmdHdlrCustomHandler, get_db_statement, + &db_statement, STD_LOADABLE_MODULE_ID)); + ENDmodInit diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h index b0e70917..92bcacab 100644 --- a/plugins/omoracle/omoracle.h +++ b/plugins/omoracle/omoracle.h @@ -20,4 +20,6 @@ enum { MAX_BUFSIZE = 2048 }; +#define BIND_MARK ':' + #endif -- cgit From 24fcd96203c9b8b84a8414cea19dcb3ba989c9ba Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:17 +0200 Subject: Fixed a mem leak --- plugins/omoracle/omoracle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 02f83551..12efd61c 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -310,6 +310,7 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); + free(pData->txt_statement); for (i = 0; i < pData->batch.arguments; i++) { for (j = 0; j < pData->batch.size; j++) free(pData->batch.parameters[i][j]); -- cgit From f89b761c84270cde71e0a6275ea80bb20f60d2df Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:18 +0200 Subject: Make the counting of bind parameters aware of literals. Literal strings passed in the statement may contain ':', let's not count them. --- plugins/omoracle/omoracle.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 12efd61c..b598192f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -180,14 +180,21 @@ static int oci_errors(void* handle, ub4 htype, sword status) } /** Returns the number of bind parameters for the statement given as - * an argument. */ + * an argument. It counts the number of appearances of ':', as in + * + * insert into foo(bar, baz) values(:bar, :baz) + * + * while taking in account that string literals must not be parsed. */ static int count_bind_parameters(char* p) { int n = 0; + int enable = 1; for (; *p; p++) - if (*p == BIND_MARK) + if (enable && *p == BIND_MARK ) n++; + else if (*p == '\'') + enable ^= 1; dbgprintf ("omoracle statement has %d parameters\n", n); return n; } @@ -429,7 +436,6 @@ CODESTARTdoAction } for (i = 0; i < pData->batch.arguments && params[i]; i++) { - dbgprintf ("omoracle argument on batch[%d][%d]: %s\n", i, n, params[i]); pData->batch.parameters[i][n] = strdup(params[i]); CHKmalloc(pData->batch.parameters[i][n]); } -- cgit From 9a897329ec6f80c99ca039f12388961417e0a422 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:19 +0200 Subject: Add some debugging output --- plugins/omoracle/omoracle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index b598192f..7199f3e1 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -436,6 +436,7 @@ CODESTARTdoAction } for (i = 0; i < pData->batch.arguments && params[i]; i++) { + dbgprintf("batch[%d][%d]=%s\n", i, n, params[i]); pData->batch.parameters[i][n] = strdup(params[i]); CHKmalloc(pData->batch.parameters[i][n]); } -- cgit From ca28204f7ba5c5c520b52180e26471e12af83560 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:20 +0200 Subject: Add the callback for OCIBindDynamic. Let's hope it works. --- plugins/omoracle/omoracle.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 7199f3e1..48b97b27 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -179,6 +179,34 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_ERROR; } +/** Callback for OCIBindDynamic. + * + * OCI doesn't insert an array of char* by itself (although it can + * handle arrays of int), so we must either run in batches of size one + * (no way) or bind all parameters with OCI_DATA_AT_EXEC instead of + * OCI_DEFAULT, and then give this function as an argument to + * OCIBindDynamic so that it is able to handle all strings in a single + * server trip. + * + * See the documentation of OCIBindDynamic + * (http://download.oracle.com/docs/cd/B28359_01/appdev.111/b28395/oci16rel003.htm#i444015) + * for more details. + */ +static int __attribute__((unused)) +bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, + int iter, int __attribute__((unused)) idx, + char** out, int* buflen, char* piece, + void** bd) +{ + dbgprintf ("Bound line: %s\n", in[iter]); + *out = in[iter]; + *buflen = sizeof (OCILobLocator*); + *piece = OCI_ONE_PIECE; + *bd = NULL; + return OCI_CONTINUE; +} + + /** Returns the number of bind parameters for the statement given as * an argument. It counts the number of appearances of ':', as in * -- cgit From 668f9a79fb7268f7935d93249cf283664662996d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Wed, 15 Apr 2009 16:53:21 +0200 Subject: Fixing the batch insertions. Previous versions inserted garbage (the pointer was interpreted as the string itself). It seems inserting arrays of strings is not that easy with OCI. This approach consumes 2KB per entry in the batch, so if you have batches of size 1000 you'll be using 2MB for the batch. This size doesn't change, anyways and the risk of leaking memory is gone. OCI doesn't deal well with batches of strings. :( --- plugins/omoracle/omoracle.c | 63 ++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 48b97b27..ddcb2ffa 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -192,15 +192,15 @@ static int oci_errors(void* handle, ub4 htype, sword status) * (http://download.oracle.com/docs/cd/B28359_01/appdev.111/b28395/oci16rel003.htm#i444015) * for more details. */ -static int __attribute__((unused)) -bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, - int iter, int __attribute__((unused)) idx, - char** out, int* buflen, char* piece, +static int bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, + int iter, int __attribute__((unused)) idx, + char** out, int* buflen, unsigned char* piece, void** bd) { - dbgprintf ("Bound line: %s\n", in[iter]); *out = in[iter]; - *buflen = sizeof (OCILobLocator*); + *buflen = strlen(*out) + 1; + dbgprintf ("omoracle bound line %d, length %d: %s\n", iter, *buflen, + *out); *piece = OCI_ONE_PIECE; *bd = NULL; return OCI_CONTINUE; @@ -239,16 +239,22 @@ static int prepare_statement(instanceData* pData) pData->txt_statement, strlen(pData->txt_statement), OCI_NTV_SYNTAX, OCI_DEFAULT)); - for (i = 0; i < pData->batch.arguments; i++) + for (i = 0; i < pData->batch.arguments; i++) { CHECKERR(pData->error, OCIBindByPos(pData->statement, pData->batch.bindings+i, - pData->error, - i+1, - pData->batch.parameters[i], - sizeof (OCILobLocator*), + pData->error, i+1, NULL, + MAX_BUFSIZE * + sizeof ***pData->batch.parameters, SQLT_STR, NULL, NULL, NULL, - 0, 0, OCI_DEFAULT)); + 0, 0, OCI_DATA_AT_EXEC)); + CHECKERR(pData->error, + OCIBindDynamic(pData->batch.bindings[i], + pData->error, + pData->batch.parameters[i], + bind_dynamic, NULL, NULL)); + } + finalize_it: RETiRet; } @@ -256,7 +262,7 @@ finalize_it: /* Resource allocation */ BEGINcreateInstance - int i; + int i, j; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -284,13 +290,25 @@ CODESTARTcreateInstance /* I know, this can be done with a single malloc() call but this is * easier to read. :) */ - pData->batch.parameters = malloc(pData->batch.arguments * + pData->batch.parameters = calloc(pData->batch.arguments, sizeof *pData->batch.parameters); CHKmalloc(pData->batch.parameters); for (i = 0; i < pData->batch.arguments; i++) { pData->batch.parameters[i] = calloc(pData->batch.size, sizeof **pData->batch.parameters); CHKmalloc(pData->batch.parameters[i]); + for (j = 0; j < pData->batch.size; j++) { + /* Each entry has at most MAX_BUFSIZE bytes + * because OCI doesn't like null-terminated + * strings when operating with batches, and + * the maximum size of each entry must be + * provided when binding + * parameters. MAX_BUFSIZE is long enough for + * usual entries. */ + pData->batch.parameters[i][j] = calloc(MAX_BUFSIZE, + sizeof ***pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i][j]); + } } pData->batch.bindings = calloc(pData->batch.arguments, @@ -305,7 +323,6 @@ ENDcreateInstance static int insert_to_db(instanceData* pData) { DEFiRet; - int i, j, n; CHECKERR(pData->error, OCIStmtExecute(pData->service, @@ -316,12 +333,6 @@ static int insert_to_db(instanceData* pData) CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); - for (i = 0; i < pData->batch.arguments; i++) - for (j = 0; j < pData->batch.n; j++) { - free(pData->batch.parameters[i][j]); - pData->batch.parameters[i][j] = NULL; - } - pData->batch.n = 0; finalize_it: dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? @@ -454,19 +465,19 @@ ENDparseSelectorAct BEGINdoAction int i; - int n = pData->batch.n; char **params = (char**) ppString[0]; CODESTARTdoAction - if (n == pData->batch.size) { + if (pData->batch.n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); CHKiRet(insert_to_db(pData)); } for (i = 0; i < pData->batch.arguments && params[i]; i++) { - dbgprintf("batch[%d][%d]=%s\n", i, n, params[i]); - pData->batch.parameters[i][n] = strdup(params[i]); - CHKmalloc(pData->batch.parameters[i][n]); + dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); + strncpy(pData->batch.parameters[i][pData->batch.n], params[i], + MAX_BUFSIZE); + CHKmalloc(pData->batch.parameters[i][pData->batch.n]); } pData->batch.n++; -- cgit From 3c886026be31a6af61a1b86773634c7bc4a44d7e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 16:55:25 +0200 Subject: added some doc for omoracle --- doc/Makefile.am | 1 + doc/modules.html | 5 ++- doc/omoracle.html | 78 +++++++++++++++++++++++++++++++++++++++++++ doc/rsyslog_conf_modules.html | 5 +-- 4 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 doc/omoracle.html diff --git a/doc/Makefile.am b/doc/Makefile.am index 3015d6b5..0f3dca70 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -33,6 +33,7 @@ html_files = \ dev_queue.html \ omsnmp.html \ ommysql.html \ + omoracle.html \ omlibdbi.html \ imfile.html \ imtcp.html \ diff --git a/doc/modules.html b/doc/modules.html index 92887508..4eae6db3 100644 --- a/doc/modules.html +++ b/doc/modules.html @@ -4,9 +4,8 @@

    About rsyslog Modules

    -

    Written by - Rainer - Gerhards (2007-07-28)

    +

    Written by +Rainer Gerhards (2007-07-28)

    This document is incomplete. The module interface is also quite incomplete and under development. Do not currently use it! You may want to visit Rainer's blog diff --git a/doc/omoracle.html b/doc/omoracle.html new file mode 100644 index 00000000..40f6360f --- /dev/null +++ b/doc/omoracle.html @@ -0,0 +1,78 @@ + + + +Oracle Database Output Module + + + +rsyslog module reference + +

    Oracle Database Output Module

    +

    Module Name:    omoracle

    +

    Author: Luis Fernando Muñoz Mejías <Luis.Fernando.Munoz.Mejias@cern.ch>

    +

    Available since: : 4.3.0 +

    Status: : contributed module, not maitained by rsyslog core authors +

    Description:

    +

    This module provides native support for logging to Oracle databases. It offers +superior performance over the more generic omlibdbi module. +It also includes a number of enhancements, most importantly prepared statements and +batching, what provides a big performance improvements. +

    +

    Note that this module is maintained by its original author. If you need assistance with it, +it is suggested to post questions to the +rsyslog mailing list. +

    From the header comments of this module: +

    +
+    This is an output module feeding directly to an Oracle
+    database. It uses Oracle Call Interface, a propietary module
+    provided by Oracle.
+
+    Selector lines to be used are of this form:
+
+    :omoracle:;TemplateName
+
+    The module gets its configuration via rsyslog $... directives,
+    namely:
+
+    $OmoracleDBUser: user name to log in on the database.
+
+    $OmoracleDBPassword: password to log in on the database.
+
+    $OmoracleDB: connection string (an Oracle easy connect or a db
+    name as specified by tnsnames.ora)
+
+    $OmoracleBatchSize: Number of elements to send to the DB on each
+    transaction.
+
+    $OmoracleStatement: Statement to be prepared and executed in
+    batches. Please note that Oracle's prepared statements have their
+    placeholders as ':identifier', and this module uses the colon to
+    guess how many placeholders there will be.
+
+    All these directives are mandatory. The dbstring can be an Oracle
+    easystring or a DB name, as present in the tnsnames.ora file.
+
+    The form of the template is just a list of strings you want
+    inserted to the DB, for instance:
+
+    $template TestStmt,"%hostname%%msg%"
+
+    Will provide the arguments to a statement like
+
+    $OmoracleStatement \
+        insert into foo(hostname,message)values(:host,:message)
+
+    Also note that identifiers to placeholders are arbitrarry. You
+    need to define the properties on the template in the correct order
+    you want them passed to the statement!
+
    +

    [rsyslog.conf overview] +[manual index] [rsyslog site]

    +

    This documentation is part of the +rsyslog +project.
    +Copyright © 2008, 2009 by Rainer Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

    + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index a281d9e7..df9abeea 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -19,6 +19,7 @@ generic database output module (Firebird/Interbase, MS SQL, Sybase, SQLLite, Ingres, Oracle, mSQL)
  • ommail - permits rsyslog to alert folks by mail if something important happens
    • +
  • omoracle - output module for Oracle (native OCI interface)
  • imfile -  input module for text files
  • imrelp - RELP @@ -44,9 +45,9 @@ only available if it has been loaded (using $ModLoad).

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    -- cgit From 2d5e8ba7cd05a95f897506e51dcc5adb06dbcaa8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 17:26:07 +0200 Subject: added a new error code for too-old rsyslog core which can be emittend when plugin can not load due to missing core functionality. --- plugins/omoracle/omoracle.c | 2 +- runtime/rsyslog.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ddcb2ffa..71cc8e1f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -560,7 +560,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); CHKiRet((*supported_options)(&opts)); if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, eCmdHdlrCustomHandler, get_db_statement, diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index cea457d8..8e181b9e 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -262,6 +262,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ + RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From 9348f8b9c2b5ba806dab8c7337877ab9da3f85b1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:33:14 +0200 Subject: preparing for 4.3.0 release --- ChangeLog | 7 ++++++- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 90b351f5..d01f3cb0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? +Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program and feed it (via its stdin) with syslog messages. If the program terminates, it is restarted. @@ -14,6 +14,11 @@ Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? - added $MaxOpenFiles configuration directive - bugfix: solved potential memory leak in msg processing, could manifest itself in imtcp +- bugfix: ompgsql did not detect problems in sql command execution + this could cause loss of messages. The handling was correct if the + connection broke, but not if there was a problem with statement + execution. The most probable case for such a case would be invalid + sql inside the template, and this is now much easier to diagnose. --------------------------------------------------------------------------- Version 4.1.7 [BETA] (rgerhards), 2009-04-?? - bugfix: $InputTCPMaxSessions config directive was accepted, but not diff --git a/configure.ac b/configure.ac index c7f1d532..6daf3f5a 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.3.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index 51271701..ae90f780 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 4.1.6 (devel branch) of rsyslog. +

    This documentation is for version 4.3.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit From dc777849fcd5c300f229c44daf4278d8b6842b71 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:37:41 +0200 Subject: bugfix: missing header (platform compatibility issue) --- tests/tcpflood.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 9c17fd5b..8dbc201b 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include -- cgit From 09ca443377ec85364160e965920ab6f56a374d88 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 15:10:06 +0200 Subject: update project status & cleanup removed some warning in imklog compilation, but may not have solved a lurking issue (but placed comment so that we know if something surfaces) --- doc/status.html | 16 ++++++++-------- plugins/imklog/ksym_mod.c | 42 ++++++++++++++++++++++-------------------- plugins/imklog/ksyms.h | 4 ++-- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/doc/status.html b/doc/status.html index fff3a6ff..01a9cf8f 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,22 +2,22 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2009-04-03.

    +

    This page reflects the status as of 2009-04-17.

    Current Releases

    -

    development: 4.1.6 [2009-04-07] - -change log - -download +

    development: 4.3.0 [2009-04-17] - +change log - +download
    beta: 3.21.11 [2009-04-03] - change log - download

    -

    v3 stable: 3.20.5 [2009-04-02] - change log - -download +

    v3 stable: 3.20.6 [2009-04-16] - change log - +download -
    v2 stable: 2.0.6 [2008-08-07] - change log - -download +
    v2 stable: 2.0.7 [2009-04-14] - change log - +download
    v0 and v1 are deprecated and no longer supported. If you absolutely do not like to upgrade, you may consider purchasing a commercial rsyslog support package. Just let us point diff --git a/plugins/imklog/ksym_mod.c b/plugins/imklog/ksym_mod.c index 6e48e89e..be5fdee9 100644 --- a/plugins/imklog/ksym_mod.c +++ b/plugins/imklog/ksym_mod.c @@ -1,9 +1,8 @@ -/* - * ksym_mod.c - functions for building symbol lookup tables for klogd +/* ksym_mod.c - functions for building symbol lookup tables for klogd * Copyright (c) 1995, 1996 Dr. G.W. Wettstein * Copyright (c) 1996 Enjellic Systems Development * Copyright (c) 1998-2007 Martin Schulze - * Copyright (C) 2007-2008 Rainer Gerhards + * Copyright (C) 2007-2009 Rainer Gerhards * * This file is part of rsyslog. * @@ -83,7 +82,6 @@ * Changed llseek() to lseek64() in order to skip a libc warning. */ - /* Includes. */ #include "config.h" #include @@ -112,7 +110,7 @@ #define KSYMS "/proc/kallsyms" static int num_modules = 0; -struct Module *sym_array_modules = (struct Module *) 0; +struct Module *sym_array_modules = (struct Module *) NULL; static int have_modules = 0; @@ -266,7 +264,7 @@ static void FreeModules() } free(sym_array_modules); - sym_array_modules = (struct Module *) 0; + sym_array_modules = (struct Module *) NULL; num_modules = 0; return; } @@ -390,11 +388,11 @@ static int AddSymbol(line) mp->sym_array = (struct sym_table *) realloc(mp->sym_array, \ (mp->num_syms+1) * sizeof(struct sym_table)); - if ( mp->sym_array == (struct sym_table *) 0 ) + if ( mp->sym_array == (struct sym_table *) NULL ) return(0); mp->sym_array[mp->num_syms].name = strdup(p); - if ( mp->sym_array[mp->num_syms].name == (char *) 0 ) + if ( mp->sym_array[mp->num_syms].name == (char *) NULL ) return(0); /* Stuff interesting information into the module. */ @@ -424,15 +422,21 @@ static int AddSymbol(line) * If a match cannot be found a diagnostic string is printed. * If a match is found the pointer to the symbolic name most * closely matching the address is returned. + * + * TODO: We are using int values for the offset, but longs for the value + * values. This may create some trouble in the future (on 64 Bit OS?). + * Anyhow, I have not changed this, because we do not seem to have any + * issue and my understanding of this code is limited (and I don't see + * need to invest more time to dig much deeper). + * rgerhards, 2009-04-17 **************************************************************************/ extern char * LookupModuleSymbol(value, sym) unsigned long value; struct symbol *sym; { - auto int nmod, - nsym; - auto struct sym_table *last; - auto struct Module *mp; + int nmod, nsym; + struct sym_table *last; + struct Module *mp; static char ret[100]; sym->size = 0; @@ -443,8 +447,7 @@ extern char * LookupModuleSymbol(value, sym) for (nmod = 0; nmod < num_modules; ++nmod) { mp = &sym_array_modules[nmod]; - /* - * Run through the list of symbols in this module and + /* Run through the list of symbols in this module and * see if the address can be resolved. */ for(nsym = 1, last = &mp->sym_array[0]; @@ -453,13 +456,12 @@ extern char * LookupModuleSymbol(value, sym) if ( mp->sym_array[nsym].value > value ) { if ( sym->size == 0 || - (value - last->value) < sym->offset || - ( (sym->offset == (value - last->value)) && - (mp->sym_array[nsym].value-last->value) < sym->size ) ) + (int) (value - last->value) < sym->offset || + ( (sym->offset == (int) (value - last->value)) && + (int) (mp->sym_array[nsym].value-last->value) < sym->size ) ) { sym->offset = value - last->value; - sym->size = mp->sym_array[nsym].value - \ - last->value; + sym->size = mp->sym_array[nsym].value - last->value; ret[sizeof(ret)-1] = '\0'; if ( mp->name == NULL ) snprintf(ret, sizeof(ret)-1, @@ -478,5 +480,5 @@ extern char * LookupModuleSymbol(value, sym) return(ret); /* It has been a hopeless exercise. */ - return((char *) 0); + return(NULL); } diff --git a/plugins/imklog/ksyms.h b/plugins/imklog/ksyms.h index b5362ff3..a168947b 100644 --- a/plugins/imklog/ksyms.h +++ b/plugins/imklog/ksyms.h @@ -2,7 +2,7 @@ * Copyright (c) 1995, 1996 Dr. G.W. Wettstein * Copyright (c) 1996 Enjellic Systems Development * Copyright (c) 2004-7 Martin Schulze - * Copyright (c) 2007-2008 Rainer Gerhards + * Copyright (c) 2007-2009 Rainer Gerhards * * This file is part of rsyslog. * @@ -26,7 +26,7 @@ struct symbol { - char *name; + uchar *name; int size; int offset; }; -- cgit From 889a0a1da8b2fb74b04647a345f64fce6c36708f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 15:19:57 +0200 Subject: some cleanup ... mostly removal of compile-time warnings (thanks to Michael Biebl for suggesting to look after that) --- ChangeLog | 2 ++ plugins/imgssapi/imgssapi.c | 1 - plugins/omgssapi/omgssapi.c | 2 +- runtime/nsd_gtls.c | 2 ++ tools/msggen.c | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index d01f3cb0..48343b5f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ --------------------------------------------------------------------------- +Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +--------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program and feed it (via its stdin) with syslog messages. If the program diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index debe935e..b9d7dfe3 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -249,7 +249,6 @@ onErrClose(tcps_sess_t *pSess) static rsRetVal doOpenLstnSocks(tcpsrv_t *pSrv) { - int *pRet = NULL; gsssrv_t *pGSrv; DEFiRet; diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index e0cc8af6..361f657f 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -444,7 +444,7 @@ CODESTARTdoAction /* error! */ dbgprintf("error forwarding via tcp, suspending\n"); pData->eDestState = eDestFORW_SUSP; - iRet = RS_RET_SUSPENDED; + ABORT_FINALIZE(RS_RET_SUSPENDED); } break; } diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 3a79a015..5786e191 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -82,6 +82,7 @@ static gnutls_certificate_credentials xcred; static gnutls_dh_params dh_params; #ifdef DEBUG +#if 0 /* uncomment, if needed some time again -- DEV Debug only */ /* This defines a log function to be provided to GnuTLS. It hopefully * helps us track down hard to find problems. * rgerhards, 2008-06-20 @@ -90,6 +91,7 @@ static void logFunction(int level, const char *msg) { dbgprintf("GnuTLS log msg, level %d: %s\n", level, msg); } +#endif #endif /* #ifdef DEBUG */ diff --git a/tools/msggen.c b/tools/msggen.c index 7990a3c8..06244c18 100644 --- a/tools/msggen.c +++ b/tools/msggen.c @@ -24,7 +24,7 @@ #include #include -int main(int argc, char *argv[]) +int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) { int i; -- cgit From 1fb5cee04dfd4d40e64e26a0c622640781cd06f7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 17:53:33 +0200 Subject: improved testbench - added tests for queue disk-only mode --- ChangeLog | 2 ++ tests/Makefile.am | 4 +++- tests/diskqueue.sh | 31 +++++++++++++++++++++++++++++++ tests/testsuites/diskqueue.conf | 16 ++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 tests/diskqueue.sh create mode 100644 tests/testsuites/diskqueue.conf diff --git a/ChangeLog b/ChangeLog index 48343b5f..85411722 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- improved testbench + * added tests for queue disk-only mode (checks disk queue logic) --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/tests/Makefile.am b/tests/Makefile.am index 87dca985..0f4cbce1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh diskqueue.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -27,6 +27,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ parsertest.sh \ + diskqueue.sh \ + testsuites/diskqueue.conf \ manytcp.sh \ testsuites/manytcp.conf \ omod-if-array.sh \ diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh new file mode 100755 index 00000000..6384eb64 --- /dev/null +++ b/tests/diskqueue.sh @@ -0,0 +1,31 @@ +# Test for disk-only queue mode +# This test checks if queue files can be correctly written +# and read back, but it does not test the transition from +# memory to disk mode for DA queues. +# added 2009-04-17 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo testing queue disk-only mode +rm -rf test-spool +mkdir test-spool +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# 20000 messages should be enough - the disk test is slow enough ;) +./tcpflood 127.0.0.1 13514 1 20000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 4 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 19999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +rm -rf test-spool diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf new file mode 100644 index 00000000..8851a459 --- /dev/null +++ b/tests/testsuites/diskqueue.conf @@ -0,0 +1,16 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit From aa43d7f83125f20f8efdf4152bdd9a09e7a81495 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Apr 2009 16:32:33 +0200 Subject: doc: added (hopefully) easier to grasp queue explanation --- ChangeLog | 2 + doc/Makefile.am | 16 ++- doc/dataflow.png | Bin 0 -> 24601 bytes doc/direct_queue0.png | Bin 0 -> 2048 bytes doc/direct_queue1.png | Bin 0 -> 2979 bytes doc/direct_queue2.png | Bin 0 -> 4117 bytes doc/direct_queue3.png | Bin 0 -> 4430 bytes doc/direct_queue_directq.png | Bin 0 -> 10075 bytes doc/direct_queue_rsyslog.png | Bin 0 -> 10465 bytes doc/direct_queue_rsyslog2.png | Bin 0 -> 12350 bytes doc/queue_analogy_tv.png | Bin 0 -> 18730 bytes doc/queues.html | 14 ++- doc/queues_analogy.html | 259 ++++++++++++++++++++++++++++++++++++++ doc/src/dataflow.dia | Bin 0 -> 2662 bytes doc/src/direct_queue0.dia | Bin 0 -> 966 bytes doc/src/direct_queue1.dia | Bin 0 -> 1058 bytes doc/src/direct_queue2.dia | Bin 0 -> 1143 bytes doc/src/direct_queue3.dia | Bin 0 -> 1214 bytes doc/src/direct_queue_directq.dia | Bin 0 -> 1705 bytes doc/src/direct_queue_rsyslog.dia | Bin 0 -> 2311 bytes doc/src/direct_queue_rsyslog2.dia | Bin 0 -> 2638 bytes doc/src/queue_analogy_tv.dia | Bin 0 -> 1749 bytes 22 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 doc/dataflow.png create mode 100644 doc/direct_queue0.png create mode 100644 doc/direct_queue1.png create mode 100644 doc/direct_queue2.png create mode 100644 doc/direct_queue3.png create mode 100644 doc/direct_queue_directq.png create mode 100644 doc/direct_queue_rsyslog.png create mode 100644 doc/direct_queue_rsyslog2.png create mode 100644 doc/queue_analogy_tv.png create mode 100644 doc/queues_analogy.html create mode 100644 doc/src/dataflow.dia create mode 100644 doc/src/direct_queue0.dia create mode 100644 doc/src/direct_queue1.dia create mode 100644 doc/src/direct_queue2.dia create mode 100644 doc/src/direct_queue3.dia create mode 100644 doc/src/direct_queue_directq.dia create mode 100644 doc/src/direct_queue_rsyslog.dia create mode 100644 doc/src/direct_queue_rsyslog2.dia create mode 100644 doc/src/queue_analogy_tv.dia diff --git a/ChangeLog b/ChangeLog index 85411722..976d8084 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- improved doc + * added (hopefully) easier to grasp queue explanation - improved testbench * added tests for queue disk-only mode (checks disk queue logic) --------------------------------------------------------------------------- diff --git a/doc/Makefile.am b/doc/Makefile.am index 0f3dca70..4d9d94ff 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -109,6 +109,20 @@ html_files = \ rsyslog_conf_output.html \ rsyslog_conf_templates.html \ rsyslog_conf_nomatch.html \ + queues_analogy.html \ src/classes.dia -EXTRA_DIST = $(html_files) +grfx_files = \ + direct_queue0.png \ + direct_queue1.png \ + direct_queue2.png \ + direct_queue3.png \ + direct_queue_rsyslog.png \ + direct_queue_rsyslog2.png \ + direct_queue_directq.png \ + dataflow.png \ + queue_analogy_tv.png \ + gssapi.png \ + rsyslog-vers.png + +EXTRA_DIST = $(html_files) $(grfx_files) diff --git a/doc/dataflow.png b/doc/dataflow.png new file mode 100644 index 00000000..fd614d8c Binary files /dev/null and b/doc/dataflow.png differ diff --git a/doc/direct_queue0.png b/doc/direct_queue0.png new file mode 100644 index 00000000..6d1b373f Binary files /dev/null and b/doc/direct_queue0.png differ diff --git a/doc/direct_queue1.png b/doc/direct_queue1.png new file mode 100644 index 00000000..503f8151 Binary files /dev/null and b/doc/direct_queue1.png differ diff --git a/doc/direct_queue2.png b/doc/direct_queue2.png new file mode 100644 index 00000000..cbb99af8 Binary files /dev/null and b/doc/direct_queue2.png differ diff --git a/doc/direct_queue3.png b/doc/direct_queue3.png new file mode 100644 index 00000000..cc49299f Binary files /dev/null and b/doc/direct_queue3.png differ diff --git a/doc/direct_queue_directq.png b/doc/direct_queue_directq.png new file mode 100644 index 00000000..c5d8769d Binary files /dev/null and b/doc/direct_queue_directq.png differ diff --git a/doc/direct_queue_rsyslog.png b/doc/direct_queue_rsyslog.png new file mode 100644 index 00000000..6150222d Binary files /dev/null and b/doc/direct_queue_rsyslog.png differ diff --git a/doc/direct_queue_rsyslog2.png b/doc/direct_queue_rsyslog2.png new file mode 100644 index 00000000..807b064d Binary files /dev/null and b/doc/direct_queue_rsyslog2.png differ diff --git a/doc/queue_analogy_tv.png b/doc/queue_analogy_tv.png new file mode 100644 index 00000000..fedcb558 Binary files /dev/null and b/doc/queue_analogy_tv.png differ diff --git a/doc/queues.html b/doc/queues.html index 41c5865f..4a9509a0 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -1,6 +1,5 @@ - Understanding rsyslog queues back @@ -10,6 +9,13 @@ queue, one part of the system "produces" something while another part "consumes" this something. The "something" is most often syslog messages, but queues may also be used for other purposes.

    +

    This document provides a good insight into technical details, operation modes +and implications. In addition to it, an +rsyslog queue concepts overview document +exists which tries to explain queues with the help of some analogies. This may +probably be a better place to start reading about queues. I assume that once you +have understood that document, the material here will be much easier to grasp +and look much more natural.

    The most prominent example is the main message queue. Whenever rsyslog receives a message (e.g. locally, via UDP, TCP or in whatever else way), it places these messages into the main message queue. Later, it is dequeued by the @@ -18,7 +24,7 @@ front of each action, there is also a queue, which potentially de-couples the filter processing from the actual action (e.g. writing to file, database or forwarding to another host).

    Where are Queues Used?

    -

     Currently, queues are used for the main message queue and for the +

    Currently, queues are used for the main message queue and for the actions.

    There is a single main message queue inside rsyslog. Each input module delivers messages to it. The main message queue worker filters messages based on @@ -354,8 +360,8 @@ save.

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html new file mode 100644 index 00000000..1584c66d --- /dev/null +++ b/doc/queues_analogy.html @@ -0,0 +1,259 @@ + + +turning lanes and rsyslog queues - an analogy + +back + +

    Turning Lanes and Rsyslog Queues - an Analogy

    +

    If there is a single object absolutely vital to understanding the way +rsyslog works, this object is queues. Queues offer a variety of services, +including support for multithreading. While there is elaborate in-depth +documentation on the ins and outs of rsyslog queues, +some of the concepts are hard to grasp even for experienced people. I think this +is because rsyslog uses a very high layer of abstraction which includes things +that look quite unnatural, like queues that do not actually queue... +

    With this document, I take a different approach: I will not describe every specific +detail of queue operation but hope to be able to provide the core idea of how +queues are used in rsyslog by using an analogy. I will compare the rsyslog data flow +with real-life traffic flowing at an intersection. +

    But first let's set the stage for the rsyslog part. The graphic below describes +the data flow inside rsyslog: +

    rsyslog data flow +

    Note that there is a video tutorial +available on the data flow. It is not perfect, but may aid in understanding this picture. +

    For our needs, the important fact to know is that messages enter rsyslog on "the +left side" (for example, via UDP), are being preprocessed, put into the +so-called main queue, taken off that queue, be filtered and be placed into one or +several action queues (depending on filter results). They leave rsyslog on "the +right side" where output modules (like the file or database writer) consume them. +

    So there are always two stages where a message (conceptually) is queued - first +in the main queue and later on in n action specific queues (with n being the number of +actions that the message in question needs to be processed by, what is being decided +by the "Filter Engine"). As such, a message will be in at least two queues +during its lifetime (with the exeception of messages being discarded by the queue itself, +but for the purpose of this document, we will ignore that possibility). +

    Also, it is vitally +important to understand that each action has a queue sitting in front of it. +If you have dug into the details of rsyslog configuration, you have probably seen that +a queue mode can be set for each action. And the default queue mode is the so-called +"direct mode", in which "the queue does not actually enqueue data". +That sounds silly, but is not. It is an important abstraction that helps keep the code clean. +

    To understand this, we first need to look at who is the active component. In our data flow, +the active part always sits to the left of the object. For example, the "Preprocessor" +is being called by the inputs and calls itself into the main message queue. That is, the queue +receiver is called, it is passive. One might think that the "Parser & Filter Engine" +is an active component that actively pulls messages from the queue. This is wrong! Actually, +it is the queue that has a pool of worker threads, and these workers pull data from the queue +and then call the passively waiting Parser and Filter Engine with those messages. So the +main message queue is the active part, the Parser and Filter Engine is passive. +

    Let's now try an anlogy analogy for this part: Think about a TV show. The show is produced +in some TV studio, from there sent (actively) to a radio tower. The radio tower passively +receives from the studio and then actively sends out a signal, which is passively received +by your TV set. In our simplified view, we have the following picture: +

    rsyslog queues and TV analogy +

    The lower part of the picture lists the equivalent rsyslog entities, in an abstracted way. +Every queue has a producer (in the above sample the input) and a consumer (in the above sample the Parser +and Filter Engine). Their active and passive functions are equivalent to the TV entities +that are listed on top of the rsyslog entity. For example, a rsyslog consumer can never +actively initate reception of a message in the same way a TV set can not actively +"initiate" a TV show - both can only "handle" (display or process) +what is sent to them. +

    Now let's look at the action queues: here, the active part, the producer, is the +Parser and Filter Engine. The passive part is the Action Processor. The later does any +processing that is necessary to call the output plugin, in particular it processes the template +to create the plugin calling parameters (eiter a string or vector of arguments). From the +action queue's point of view, Action Processor and Output form a single entity. Again, the +TV set analogy holds. The Output does not actively ask the queue for data, but +rater passively waits until the queue itself pushes some data to it. + +

    Armed with this knowledge, we can now look at the way action queue modes work. My analogy here +is a junction, as shown below (note that the colors in the pictures below are not related to +the colors in the pictures above!): +

    +

    This is a very simple real-life traffic case: one road joins another. We look at +traffic on the straight road, here shown by blue and green arrows. Traffic in the +opposing direction is shown in blue. Traffic flows without +any delays as long as nobody takes turns. To be more precise, if the opposing traffic takes +a (right) turn, traffic still continues to flow without delay. However, if a car in the red traffic +flow intend to do a (left, then) turn, the situation changes: +

    +

    The turning car is represented by the green arrow. It can not turn unless there is a gap +in the "blue traffic stream". And as this car blocks the roadway, the remaining +traffic (now shown in red, which should indicate the block condition), +must wait until the "green" car has made its turn. So +a queue will build up on that lane, waiting for the turn to be completed. +Note that in the examples below I do not care that much about the properties of the +opposing traffic. That is, because its structure is not really important for what I intend to +show. Think about the blue arrow as being a traffic stream that most of the time blocks +left-turners, but from time to time has a gap that is sufficiently large for a left-turn +to complete. +

    Our road network designers know that this may be unfortunate, and for more important roads +and junctions, they came up with the concept of turning lanes: +

    +

    Now, the car taking the turn can wait in a special area, the turning lane. As such, +the "straight" traffic is no longer blocked and can flow in parallel to the +turning lane (indicated by a now-green-again arrow). + +

    However, the turning lane offers only finite space. So if too many cars intend to +take a left turn, and there is no gap in the "blue" traffic, we end up with +this well-known situation: +

    +

    The turning lane is now filled up, resulting in a tailback of cars intending to +left turn on the main driving lane. The end result is that "straight" traffic +is again being blocked, just as in our initial problem case without the turning lane. +In essence, the turning lane has provided some relief, but only for a limited amount of +cars. Street system designers now try to weight cost vs. benefit and create (costly) +turning lanes that are sufficiently large to prevent traffic jams in most, but not all +cases. +

    Now let's dig a bit into the mathematical properties of turning lanes. We assume that +cars all have the same length. So, units of cars, the length is alsways one (which is nice, +as we don't need to care about that factor any longer ;)). A turning lane has finite capacity of +n cars. As long as the number of cars wanting to take a turn is less than or eqal +to n, "straigth traffic" is not blocked (or the other way round, traffic +is blocked if at least n + 1 cars want to take a turn!). We can now find an optimal +value for n: it is a function of the probability that a car wants to turn +and the cost of the turning lane +(as well as the probability there is a gap in the "blue" traffic, but we ignore this +in our simple sample). +If we start from some finite upper bound of n, we can decrease +n to a point where it reaches zero. But let's first look at n = 1, in which case exactly +one car can wait on the turning lane. More than one car, and the rest of the traffic is blocked. +Our everyday logic indicates that this is actually the lowest boundary for n. +

    In an abstract view, however, n can be zero and that works nicely. There still can be +n cars at any given time on the turning lane, it just happens that this means there can +be no car at all on it. And, as usual, if we have at least n + 1 cars wanting to turn, +the main traffic flow is blocked. True, but n + 1 = 0 + 1 = 1 so as soon as there is any +car wanting to take a turn, the main traffic flow is blocked (remeber, in all cases, I assume +no sufficently large gaps in the opposing trafic). +

    This is the situation our everyday perception calls "road without turning lane". +In my math model, it is a "road with turning lane of size 0". The subtle difference +is important: my math model guarantees that, in an abstract sense, there always is a turning +lane, it may just be too short. But it exists, even though we don't see it. And now I can +claim that even in my small home village, all roads have turning lanes, which is rather +impressive, isn't it? ;) +

    And now we finally have arrived at rsyslog's queues! Rsyslog action queues exists for +all actions just like all roads in my village have turning lanes! And as in this real-life sample, +it may be hard to see the action queues for that reason. In rsyslog, the "direct" queue +mode is the equivalent to the 0-sized turning lane. And actions queues are the equivalent to turning +lanes in general, with our real-life n being the maximum queue size. +The main traffic line (which sometimes is blocked) is the equivalent to the +main message queue. And the periods without gaps in the opposing traffic are equivalent to +execution time of an action. In a rough sketch, the rsyslog main and action queues look like in the +following picture. +

    +

    We need to read this picture from right to left (otherwise I would need to redo all +the graphics ;)). In action 3, you see a 0-sized turning lane, aka an action queue in "direct" +mode. All other queues are run in non-direct modes, but with different sizes greater than 0. +

    Let us first use our car analogy: +Assume we are in a car on the main lane that wants to take turn into the "action 4" +road. We pass action 1, where a number of cars wait in the turning lane and we pass +action 2, which has a slightly smaller, but still not filled up turning lane. So we pass that +without delay, too. Then we come to "action 4", which has no turning lane. Unfortunately, +the car in front of us wants to turn left into that road, so it blocks the main lane. So, this time +we need to wait. An observer standing on the sidewalk may see that while we need to wait, there are +still some cars in the "action 4" turning lane. As such, even though no new cars can +arrive on the main lane, cars still turn into the "action 4" lane. In other words, +an observer standing in "action 4" road is unable to see that traffic on the main lane +is blocked. +

    Now on to rsyslog: Other than in the real-world traffic example, messages in rsyslog +can - at more or less the +same time - "take turns" into several roads at once. This is done by duplicating the message +if the road has a non-zero-sized "turning lane" - or in rsyslog terms a queue that is +running in any non-direct mode. If so, a deep copy of the message object is made, that placed into +the action queue and then the initial message proceeds on the "main lane". The action +queue then pushes the duplicates through action processing. This is also the reason why a +discard action inside a non-direct queue does not seem to have an effect. Actually, it discards the +copy that was just created, but the original message object continues to flow. +

    +In action 1, we have some entries in the action queue, as we have in action 2 (where the queue is +slightly shorter). As we have seen, new messages pass action one and two almost instantaneously. +However, when a messages reaches action 3, its flow is blocked. Now, message processing must wait +for the action to complete. Processing flow in a direct mode queue is something like a U-turn: + +

    message processing in an rsyslog action queue in direct mode +

    The message starts to execute the action and once this is done, processing flow continues. +In a real-life analogy, this may be the route of a delivery man who needs to drop a parcel +in a side street before he continues driving on the main route. As a side-note, think of what happens +with the rest of the delivery route, at least for today, if the delivery truck has a serious accident +in the side street. The rest of the parcels won't be delivered today, will they? This is exactly how the +discard action works. It drops the message object inside the action and thus the message will no +longer be available for further delivery - but as I said, only if the discard is done in a +direct mode queue (I am stressing this example because it often causes a lot of confusion). +

    Back to the overall scenario. We have seen that messages need to wait for action 3 to +complete. Does this necessarily mean that at the same time no messages can be processed +in action 4? Well, it depends. As in the real-life scenario, action 4 will continue to +receive traffic as long as its action queue ("turn lane") is not drained. In +our drawing, it is not. So action 4 will be executed while messages still wait for action 3 +to be completed. +

    Now look at the overall picture from a slightly different angle: +

    message processing in an rsyslog action queue in direct mode +

    The number of all connected green and red arrows is four - one each for action 1, 2 and 3 +(this one is dotted as action 4 was a special case) and one for the "main lane" as +well as acton 3 (this one contains the sole red arrow). This number is the lower bound for +the number of threads in rsyslog's output system ("right-hand part" of the main message +queue)! Each of the connected arrows is a continous thread and each "turn lane" is +a place where processing is forked onto a new thread. Also, note that in action 3 the processing +is carried out on the main thread, but not in the non-direct queue modes. +

    I have said this is "the lower bound for the number of threads...". This is with +good reason: the main queue may have more than one worker thread (individual action queues +currently do not support this, but could do in the future - there are good reasons for that, too +but exploring why would finally take us away from what we intend to see). Note that you +configure an upper bound for the number of main message queue worker threads. The actual number +varies depending on a lot of operational variables, most importantly the number of messages +inside the queue. The number t_m of actually running threads is within the integer-interval +[0,confLimit] (with confLimit being the operator configured limit, which defaults to 5). +Output plugins may have more than one thread created by themselves. It is quite unusual for an +output plugin to create such threads and so I assume we do not have any of these. +Then, the overall number of threads in rsyslog's filtering and output system is +t_total = t_m + number of actions in non-direct modes. Add the number of +inputs configured to that and you have the total number of threads running in rsyslog at +a given time (assuming again that inputs utilize only one thread per plugin, a not-so-safe +assumption). +

    A quick side-note: I gave the lower bound for t_m as zero, which is somewhat in contrast +to what I wrote at the begin of the last paragraph. Zero is actually correct, because rsyslog +stops all worker threads when there is no work to do. This is also true for the action queues. +So the ultimate lower bound for a rsyslog output system without any work to carry out actually is zero. +But this bound will never be reached when there is continous flow of activity. And, if you are +curios: if the number of workers is zero, the worker wakeup process is actually handled within the +threading context of the "left-hand-side" (or producer) of the queue. After being +started, the worker begins to play the active queue component again. All of this, of course, +can be overridden with configuraton directives. +

    When looking at the threading model, one can simply add n lanes to the main lane but otherwise +retain the traffic analogy. This is a very good description of the actual process (think what this +means to the "turning lanes"; hint: there still is only one per action!). +

    Let's try to do a warp-up: I have hopefully been able to show that in rsyslog, an action +queue "sits in front of" each output plugin. Messages are received and flow, from input +to output, over various stages and two level of queues to the outputs. Actions queues are always +present, but may not easily be visible when in direct mode (where no actual queueing takes place). +The "road junktion with turning lane" analogy well describes the way - and intent - of the various +queue levels in rsyslog. +

    On the output side, the queue is the active component, not the consumer. As such, the consumer +can not ask the queue for anything (like n number of messages) but rather is activated by the queue +itself. As such, a queue somewhat resembles a "living thing" whereas the outputs are +just tools that this "living thing" uses. +

    Note that I left out a couple of subtleties, especially when it comes +to error handling and terminating +a queue (you hopefully have now at least a rough idea why I say "terminating a queue" +and not "terminating an action" - who is the "living thing"?). An action returns +a status to the queue, but it is the queue that ultimately decides which messages can finally be +considered processed and which not. Please note that the queue may even cancel an output right in +the middle of its action. This happens, if configured, if an output needs more than a configured +maximum processing time and is a guard condition to prevent slow outputs from defering a rsyslog +restart for too long. Especially in this case re-queueing and cleanup is not trivial. Also, note that +I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additonal +constraints, especially in regard to the threading model. Transitioning between actual +disk-assisted mode and pure-in-memory-mode (which is done automatically when needed) is also far from +trivial and a real joy for an implementor to work on ;). +

    If you have not done so before, it may be worth reading the +rsyslog queue user's guide, which most importantly lists all +the knobs you can turn to tweak queue operation. +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 3 or higher.

    + + diff --git a/doc/src/dataflow.dia b/doc/src/dataflow.dia new file mode 100644 index 00000000..3875fc61 Binary files /dev/null and b/doc/src/dataflow.dia differ diff --git a/doc/src/direct_queue0.dia b/doc/src/direct_queue0.dia new file mode 100644 index 00000000..4446619b Binary files /dev/null and b/doc/src/direct_queue0.dia differ diff --git a/doc/src/direct_queue1.dia b/doc/src/direct_queue1.dia new file mode 100644 index 00000000..7a64ea09 Binary files /dev/null and b/doc/src/direct_queue1.dia differ diff --git a/doc/src/direct_queue2.dia b/doc/src/direct_queue2.dia new file mode 100644 index 00000000..b0c394c0 Binary files /dev/null and b/doc/src/direct_queue2.dia differ diff --git a/doc/src/direct_queue3.dia b/doc/src/direct_queue3.dia new file mode 100644 index 00000000..bc477b25 Binary files /dev/null and b/doc/src/direct_queue3.dia differ diff --git a/doc/src/direct_queue_directq.dia b/doc/src/direct_queue_directq.dia new file mode 100644 index 00000000..37fdb44c Binary files /dev/null and b/doc/src/direct_queue_directq.dia differ diff --git a/doc/src/direct_queue_rsyslog.dia b/doc/src/direct_queue_rsyslog.dia new file mode 100644 index 00000000..9a030117 Binary files /dev/null and b/doc/src/direct_queue_rsyslog.dia differ diff --git a/doc/src/direct_queue_rsyslog2.dia b/doc/src/direct_queue_rsyslog2.dia new file mode 100644 index 00000000..c596f39f Binary files /dev/null and b/doc/src/direct_queue_rsyslog2.dia differ diff --git a/doc/src/queue_analogy_tv.dia b/doc/src/queue_analogy_tv.dia new file mode 100644 index 00000000..00fbdeb5 Binary files /dev/null and b/doc/src/queue_analogy_tv.dia differ -- cgit From 8f1fc598c7f73f5718d0dbc656f7e78c72634a9b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Apr 2009 17:01:29 +0200 Subject: doc status update & typo fixes --- doc/queues_analogy.html | 42 +++++++++++++++++++++--------------------- doc/status.html | 6 +++--- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html index 1584c66d..4f41fa26 100644 --- a/doc/queues_analogy.html +++ b/doc/queues_analogy.html @@ -30,7 +30,7 @@ right side" where output modules (like the file or database writer) consume in the main queue and later on in n action specific queues (with n being the number of actions that the message in question needs to be processed by, what is being decided by the "Filter Engine"). As such, a message will be in at least two queues -during its lifetime (with the exeception of messages being discarded by the queue itself, +during its lifetime (with the exception of messages being discarded by the queue itself, but for the purpose of this document, we will ignore that possibility).

    Also, it is vitally important to understand that each action has a queue sitting in front of it. @@ -46,7 +46,7 @@ is an active component that actively pulls messages from the queue. This is wron it is the queue that has a pool of worker threads, and these workers pull data from the queue and then call the passively waiting Parser and Filter Engine with those messages. So the main message queue is the active part, the Parser and Filter Engine is passive. -

    Let's now try an anlogy analogy for this part: Think about a TV show. The show is produced +

    Let's now try an analogy analogy for this part: Think about a TV show. The show is produced in some TV studio, from there sent (actively) to a radio tower. The radio tower passively receives from the studio and then actively sends out a signal, which is passively received by your TV set. In our simplified view, we have the following picture: @@ -55,16 +55,16 @@ by your TV set. In our simplified view, we have the following picture: Every queue has a producer (in the above sample the input) and a consumer (in the above sample the Parser and Filter Engine). Their active and passive functions are equivalent to the TV entities that are listed on top of the rsyslog entity. For example, a rsyslog consumer can never -actively initate reception of a message in the same way a TV set can not actively +actively initiate reception of a message in the same way a TV set cannot actively "initiate" a TV show - both can only "handle" (display or process) what is sent to them.

    Now let's look at the action queues: here, the active part, the producer, is the -Parser and Filter Engine. The passive part is the Action Processor. The later does any +Parser and Filter Engine. The passive part is the Action Processor. The latter does any processing that is necessary to call the output plugin, in particular it processes the template -to create the plugin calling parameters (eiter a string or vector of arguments). From the +to create the plugin calling parameters (either a string or vector of arguments). From the action queue's point of view, Action Processor and Output form a single entity. Again, the TV set analogy holds. The Output does not actively ask the queue for data, but -rater passively waits until the queue itself pushes some data to it. +rather passively waits until the queue itself pushes some data to it.

    Armed with this knowledge, we can now look at the way action queue modes work. My analogy here is a junction, as shown below (note that the colors in the pictures below are not related to @@ -75,9 +75,9 @@ traffic on the straight road, here shown by blue and green arrows. Traffic in th opposing direction is shown in blue. Traffic flows without any delays as long as nobody takes turns. To be more precise, if the opposing traffic takes a (right) turn, traffic still continues to flow without delay. However, if a car in the red traffic -flow intend to do a (left, then) turn, the situation changes: +flow intends to do a (left, then) turn, the situation changes:

    -

    The turning car is represented by the green arrow. It can not turn unless there is a gap +

    The turning car is represented by the green arrow. It cannot turn unless there is a gap in the "blue traffic stream". And as this car blocks the roadway, the remaining traffic (now shown in red, which should indicate the block condition), must wait until the "green" car has made its turn. So @@ -123,8 +123,8 @@ Our everyday logic indicates that this is actually the lowest boundary for n< n cars at any given time on the turning lane, it just happens that this means there can be no car at all on it. And, as usual, if we have at least n + 1 cars wanting to turn, the main traffic flow is blocked. True, but n + 1 = 0 + 1 = 1 so as soon as there is any -car wanting to take a turn, the main traffic flow is blocked (remeber, in all cases, I assume -no sufficently large gaps in the opposing trafic). +car wanting to take a turn, the main traffic flow is blocked (remember, in all cases, I assume +no sufficiently large gaps in the opposing traffic).

    This is the situation our everyday perception calls "road without turning lane". In my math model, it is a "road with turning lane of size 0". The subtle difference is important: my math model guarantees that, in an abstract sense, there always is a turning @@ -189,9 +189,9 @@ to be completed.

    message processing in an rsyslog action queue in direct mode

    The number of all connected green and red arrows is four - one each for action 1, 2 and 3 (this one is dotted as action 4 was a special case) and one for the "main lane" as -well as acton 3 (this one contains the sole red arrow). This number is the lower bound for +well as action 3 (this one contains the sole red arrow). This number is the lower bound for the number of threads in rsyslog's output system ("right-hand part" of the main message -queue)! Each of the connected arrows is a continous thread and each "turn lane" is +queue)! Each of the connected arrows is a continuous thread and each "turn lane" is a place where processing is forked onto a new thread. Also, note that in action 3 the processing is carried out on the main thread, but not in the non-direct queue modes.

    I have said this is "the lower bound for the number of threads...". This is with @@ -213,22 +213,22 @@ assumption). to what I wrote at the begin of the last paragraph. Zero is actually correct, because rsyslog stops all worker threads when there is no work to do. This is also true for the action queues. So the ultimate lower bound for a rsyslog output system without any work to carry out actually is zero. -But this bound will never be reached when there is continous flow of activity. And, if you are +But this bound will never be reached when there is continuous flow of activity. And, if you are curios: if the number of workers is zero, the worker wakeup process is actually handled within the threading context of the "left-hand-side" (or producer) of the queue. After being started, the worker begins to play the active queue component again. All of this, of course, -can be overridden with configuraton directives. +can be overridden with configuration directives.

    When looking at the threading model, one can simply add n lanes to the main lane but otherwise retain the traffic analogy. This is a very good description of the actual process (think what this means to the "turning lanes"; hint: there still is only one per action!).

    Let's try to do a warp-up: I have hopefully been able to show that in rsyslog, an action queue "sits in front of" each output plugin. Messages are received and flow, from input to output, over various stages and two level of queues to the outputs. Actions queues are always -present, but may not easily be visible when in direct mode (where no actual queueing takes place). -The "road junktion with turning lane" analogy well describes the way - and intent - of the various +present, but may not easily be visible when in direct mode (where no actual queuing takes place). +The "road junction with turning lane" analogy well describes the way - and intent - of the various queue levels in rsyslog.

    On the output side, the queue is the active component, not the consumer. As such, the consumer -can not ask the queue for anything (like n number of messages) but rather is activated by the queue +cannot ask the queue for anything (like n number of messages) but rather is activated by the queue itself. As such, a queue somewhat resembles a "living thing" whereas the outputs are just tools that this "living thing" uses.

    Note that I left out a couple of subtleties, especially when it comes @@ -238,12 +238,12 @@ and not "terminating an action" - who is the "living thing&quo a status to the queue, but it is the queue that ultimately decides which messages can finally be considered processed and which not. Please note that the queue may even cancel an output right in the middle of its action. This happens, if configured, if an output needs more than a configured -maximum processing time and is a guard condition to prevent slow outputs from defering a rsyslog -restart for too long. Especially in this case re-queueing and cleanup is not trivial. Also, note that -I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additonal +maximum processing time and is a guard condition to prevent slow outputs from deferring a rsyslog +restart for too long. Especially in this case re-queuing and cleanup is not trivial. Also, note that +I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additional constraints, especially in regard to the threading model. Transitioning between actual disk-assisted mode and pure-in-memory-mode (which is done automatically when needed) is also far from -trivial and a real joy for an implementor to work on ;). +trivial and a real joy for an implementer to work on ;).

    If you have not done so before, it may be worth reading the rsyslog queue user's guide, which most importantly lists all the knobs you can turn to tweak queue operation. diff --git a/doc/status.html b/doc/status.html index 01a9cf8f..c8608f73 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,7 +2,7 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2009-04-17.

    +

    This page reflects the status as of 2009-04-21.

    Current Releases

    development: 4.3.0 [2009-04-17] - @@ -13,8 +13,8 @@ change log - download

    -

    v3 stable: 3.20.6 [2009-04-16] - change log - -download +

    v3 stable: 3.22.0 [2009-04-21] - change log - +download
    v2 stable: 2.0.7 [2009-04-14] - change log - download -- cgit From 1f08d919fdd23e225e37cde83024076ad8a1197d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Apr 2009 17:32:51 +0200 Subject: typo fix (important typo) --- doc/queues_analogy.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html index 4f41fa26..e82f9d15 100644 --- a/doc/queues_analogy.html +++ b/doc/queues_analogy.html @@ -148,7 +148,7 @@ mode. All other queues are run in non-direct modes, but with different sizes gre Assume we are in a car on the main lane that wants to take turn into the "action 4" road. We pass action 1, where a number of cars wait in the turning lane and we pass action 2, which has a slightly smaller, but still not filled up turning lane. So we pass that -without delay, too. Then we come to "action 4", which has no turning lane. Unfortunately, +without delay, too. Then we come to "action 3", which has no turning lane. Unfortunately, the car in front of us wants to turn left into that road, so it blocks the main lane. So, this time we need to wait. An observer standing on the sidewalk may see that while we need to wait, there are still some cars in the "action 4" turning lane. As such, even though no new cars can -- cgit From 029218df5b7fbd6a7e4f945ecd32f84236885f53 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Apr 2009 08:38:25 +0200 Subject: another typo fix... --- doc/queues_analogy.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html index e82f9d15..d7533ad5 100644 --- a/doc/queues_analogy.html +++ b/doc/queues_analogy.html @@ -59,7 +59,7 @@ actively initiate reception of a message in the same way a TV set cannot activel "initiate" a TV show - both can only "handle" (display or process) what is sent to them.

    Now let's look at the action queues: here, the active part, the producer, is the -Parser and Filter Engine. The passive part is the Action Processor. The latter does any +Parser and Filter Engine. The passive part is the Action Processor. The later does any processing that is necessary to call the output plugin, in particular it processes the template to create the plugin calling parameters (either a string or vector of arguments). From the action queue's point of view, Action Processor and Output form a single entity. Again, the -- cgit From 42c3dcfc1ca71814e62763338a24eae8c8463069 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Apr 2009 13:39:11 +0200 Subject: performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker threads (an enhancement scheduled for all input plugins where this is possible) --- ChangeLog | 4 ++++ tcps_sess.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1ad99be1..98f59e9e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,10 @@ Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? * added tests for queue disk-only mode (checks disk queue logic) - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- performance enhancemnt: imtcp calls parser no longer on input thread + but rather inside on of the potentially many main msg queue worker + threads (an enhancement scheduled for all input plugins where this is + possible) --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/tcps_sess.c b/tcps_sess.c index d0edc018..2a3ec0df 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -43,6 +43,7 @@ #include "errmsg.h" #include "netstrm.h" #include "msg.h" +#include "datetime.h" /* static data */ @@ -50,6 +51,7 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) DEFobjCurrIf(netstrm) +DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ @@ -191,6 +193,52 @@ SetUsrP(tcps_sess_t *pThis, void *pUsr) } +/* This is a helper for submitting the message to the rsyslog core. + * It does some common processing, including resetting the various + * state variables to a "processed" state. + * Note that this function is also called if we had a buffer overflow + * due to a too-long message. So far, there is no indication this + * happened and it may be worth thinking about different handling + * of this case (what obviously would require a change to this + * function or some related code). + * rgerhards, 2009-04-23 + */ +static rsRetVal +doSubmitMessage(tcps_sess_t *pThis) +{ + msg_t *pMsg; + struct syslogTime stTime; + time_t ttGenTime; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcps_sess); + + //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + datetime.getCurrTime(&stTime, &ttGenTime); + //} + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + /* first trim the buffer to what we have actually received */ + CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); + memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); + pMsg->iLenRawMsg = pThis->iMsg; + MsgSetInputName(pMsg, (char*)pThis->pSrv->pszInputName); + MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->bParseHOSTNAME = 1; + MsgSetRcvFrom(pMsg, (char*)pThis->fromHost); + CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); + CHKiRet(submitMsg(pMsg)); + +finalize_it: + /* reset status variables */ + pThis->bAtStrtOfFram = 1; + pThis->iMsg = 0; + + RETiRet; +} + + /* This should be called before a normal (non forced) close * of a TCP session. This function checks if there is any unprocessed * message left in the TCP stream. Such a message is probably a @@ -230,9 +278,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->bAtStrtOfFram = 1; + doSubmitMessage(pThis); } finalize_it: @@ -313,9 +359,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->iMsg = 0; + doSubmitMessage(pThis); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -326,9 +370,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->iMsg = 0; + doSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -345,9 +387,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->iMsg = 0; + doSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } } @@ -432,6 +472,7 @@ CODESTARTObjClassExit(tcps_sess) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(datetime, CORE_COMPONENT); ENDObjClassExit(tcps_sess) @@ -443,6 +484,7 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ -- cgit From e65a30ddaaf027855b57355fe35b88b97053ac99 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Tue, 28 Apr 2009 21:14:50 +0200 Subject: Add licensing information. I'm not sure if GPLv3 contemplates the ability to link to proprietary software, if it was previous work. I explicitly allow linking to OCI. --- plugins/omoracle/omoracle.c | 4 ++++ plugins/omoracle/omoracle.h | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 71cc8e1f..3a1f141f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -43,6 +43,10 @@ need to define the properties on the template in the correct order you want them passed to the statement! + This file is licensed under the terms of the GPL version 3 or, at + your choice, any later version. Exceptionally (perhaps), you are + allowed to link to the Oracle Call Interface in your derived work + Author: Luis Fernando Muñoz Mejías diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h index 92bcacab..0ff879b3 100644 --- a/plugins/omoracle/omoracle.h +++ b/plugins/omoracle/omoracle.h @@ -3,6 +3,12 @@ This module needs OCI to be installed (on Red Hat-like systems this is usually the oracle-instantclient-devel RPM). + This file is part of rsyslog. + + This file is licensed under the terms of the GPL version 3 or, at + your choice, any later version. Exceptionally (perhaps), you are + allowed to link to the Oracle Call Interface in your derived work + Author: Luis Fernando Muñoz Mejías */ #ifndef __OMORACLEH__ -- cgit From 65a69831e955fa32b23e8f25d3ba67b1e3058e1d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Tue, 28 Apr 2009 21:14:51 +0200 Subject: Add the $OmoracleBatchItemSize directive This directive controls the amount of memory needed for properties in the batch. Users should specify the largest value they expect in the statement. As per Rainer's comment: on MAX_BUFSIZE: I'd tend to make this configurable, because with RFC5424 messages can be much longer and RFC5425 now recommends a minimum maximum size of 8K. So we let users to choose. Maybe we need a sensible default value to make users' lifes easier? Also, the old non-vector based interface is not supported anymore. I broke it already when moving to this stage. --- plugins/omoracle/omoracle.c | 46 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 3a1f141f..d6349d89 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -21,6 +21,13 @@ $OmoracleBatchSize: Number of elements to send to the DB on each transaction. + $OmoracleBatchItemSize: Number of characters each property may + have. Make it as big as the longest value you expect for *any* + property in the sentence. For instance, if you expect 5 arguments + to the statement, 4 have 10 bytes and the 5th may be up to 3KB, + then specify $OmoracleBatchItemSize 3072. Please, remember to + leave space to the trailing \0!! + $OmoracleStatement: Statement to be prepared and executed in batches. Please note that Oracle's prepared statements have their placeholders as ':identifier', and this module uses the colon to @@ -89,6 +96,8 @@ struct oracle_batch int n; /* Number of arguments the statement takes */ int arguments; + /** Maximum size of each parameter */ + int param_size; /* Parameters to pass to the statement on this transaction */ char*** parameters; /* Binding parameters */ @@ -125,12 +134,10 @@ static char* db_user; static char* db_password; /** Batch size. */ static int batch_size; +/** Size of each element in the batch. */ +static int batch_item_size; /** Statement to prepare and execute */ static char* db_statement; -/** Whether or not the core supports the newer array interface. The - * module is able to work in both modes, but the newer is the - * recommended one for performance reasons. */ -static int array_passing; /** Generic function for handling errors from OCI. @@ -248,8 +255,7 @@ static int prepare_statement(instanceData* pData) OCIBindByPos(pData->statement, pData->batch.bindings+i, pData->error, i+1, NULL, - MAX_BUFSIZE * - sizeof ***pData->batch.parameters, + pData->batch.param_size, SQLT_STR, NULL, NULL, NULL, 0, 0, OCI_DATA_AT_EXEC)); CHECKERR(pData->error, @@ -290,6 +296,7 @@ CODESTARTcreateInstance pData->batch.n = 0; pData->batch.size = batch_size; + pData->batch.param_size = batch_item_size * sizeof ***pData->batch.parameters; pData->batch.arguments = count_bind_parameters(pData->txt_statement); /* I know, this can be done with a single malloc() call but this is @@ -302,15 +309,14 @@ CODESTARTcreateInstance sizeof **pData->batch.parameters); CHKmalloc(pData->batch.parameters[i]); for (j = 0; j < pData->batch.size; j++) { - /* Each entry has at most MAX_BUFSIZE bytes - * because OCI doesn't like null-terminated - * strings when operating with batches, and - * the maximum size of each entry must be - * provided when binding - * parameters. MAX_BUFSIZE is long enough for - * usual entries. */ - pData->batch.parameters[i][j] = calloc(MAX_BUFSIZE, - sizeof ***pData->batch.parameters); + /* Each entry has at most + * pData->batch.param_size bytes because OCI + * doesn't like null-terminated strings when + * operating with batches, and the maximum + * size of each entry must be provided when + * binding parameters. pData->batch.param_size + * is long enough for usual entries. */ + pData->batch.parameters[i][j] = malloc(pData->batch.param_size); CHKmalloc(pData->batch.parameters[i][j]); } } @@ -480,7 +486,7 @@ CODESTARTdoAction for (i = 0; i < pData->batch.arguments && params[i]; i++) { dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); strncpy(pData->batch.parameters[i][pData->batch.n], params[i], - MAX_BUFSIZE); + pData->batch.param_size); CHKmalloc(pData->batch.parameters[i][pData->batch.n]); } pData->batch.n++; @@ -520,6 +526,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, if (db_statement != NULL) free(db_statement); db_name = db_user = db_password = db_statement = NULL; + batch_size = batch_item_size = 0; RETiRet; } @@ -549,6 +556,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbuser", 0, eCmdHdlrGetWord, NULL, &db_user, STD_LOADABLE_MODULE_ID)); @@ -563,11 +571,15 @@ CODEmodInit_QueryRegCFSLineHdlr STD_LOADABLE_MODULE_ID)); CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); CHKiRet((*supported_options)(&opts)); - if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) + if (!(opts & OMSR_TPL_AS_ARRAY)) ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, eCmdHdlrCustomHandler, get_db_statement, &db_statement, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchitemsize", 0, + eCmdHdlrInt, NULL, + &batch_item_size, STD_LOADABLE_MODULE_ID)); + ENDmodInit -- cgit From c35ce31aed4d873ed9564332374fbf82d7ff187d Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Tue, 28 Apr 2009 21:14:52 +0200 Subject: Replace get_db_statement by a template. Instead of reading a complete line, we'll use a template and delegate in the core to read such template. Then, all omoracle has to do is to find that template and use it as the prepared statement. I'm not sure if this is the correct approach, though. It has to dig too much into rsyslog's structures... txt_statement is stored in a private area, so that we don't mess too much with rsyslog's internals (I still don't feel comfortable with this much digging into template structures). --- plugins/omoracle/omoracle.c | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index d6349d89..d1b3b955 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -28,10 +28,11 @@ then specify $OmoracleBatchItemSize 3072. Please, remember to leave space to the trailing \0!! - $OmoracleStatement: Statement to be prepared and executed in - batches. Please note that Oracle's prepared statements have their - placeholders as ':identifier', and this module uses the colon to - guess how many placeholders there will be. + $OmoracleStatementTemplate: Name of the template containing the + statement to be prepared and executed in batches. Please note that + Oracle's prepared statements have their placeholders as + ':identifier', and this module uses the colon to guess how many + placeholders there will be. All these directives are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -273,6 +274,7 @@ finalize_it: /* Resource allocation */ BEGINcreateInstance int i, j; + struct template* tpl; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -289,14 +291,16 @@ CODESTARTcreateInstance CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); - pData->txt_statement = strdup(db_statement); + tpl = tplFind(db_statement, strlen(db_statement)); + pData->txt_statement = strdup(tpl->pEntryRoot->data.constant.pConstant); CHKmalloc(pData->txt_statement); dbgprintf("omoracle will run stored statement: %s\n", pData->txt_statement); pData->batch.n = 0; pData->batch.size = batch_size; - pData->batch.param_size = batch_item_size * sizeof ***pData->batch.parameters; + pData->batch.param_size = batch_item_size * + sizeof ***pData->batch.parameters; pData->batch.arguments = count_bind_parameters(pData->txt_statement); /* I know, this can be done with a single malloc() call but this is @@ -439,7 +443,6 @@ finalize_it: BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ - dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature @@ -465,6 +468,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); CHKiRet(createInstance(&pData)); CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); + CHKiRet(prepare_statement(pData)); dbgprintf ("omoracle module got all its resources allocated " @@ -530,22 +534,6 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, RETiRet; } -/** As I don't find any handler that reads an entire line, I write my - * own. */ -static int get_db_statement(char** line, char** stmt) -{ - DEFiRet; - - while (isspace(**line)) - (*line)++; - dbgprintf ("Config line: %s\n", *line); - *stmt = strdup(*line); - CHKmalloc(*stmt); - dbgprintf ("Statement: %s\n", *stmt); -finalize_it: - RETiRet; -} - BEGINmodInit() rsRetVal (*supported_options)(unsigned long *pOpts); unsigned long opts; @@ -574,8 +562,8 @@ CODEmodInit_QueryRegCFSLineHdlr if (!(opts & OMSR_TPL_AS_ARRAY)) ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); - CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, - eCmdHdlrCustomHandler, get_db_statement, + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatementtemplate", 0, + eCmdHdlrGetWord, NULL, &db_statement, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchitemsize", 0, -- cgit From 9823c73d1d07e50e6bec7f7c02c88d61d6955526 Mon Sep 17 00:00:00 2001 From: Luis Fernando Muñoz Mejías Date: Tue, 28 Apr 2009 21:14:53 +0200 Subject: Make it recover from errors on insertions. If the database rejected some entry, making the statement fail on it, the batch was not cleaned and the same values were retried over and over, causing a cascade of failures and a denial of service. We use now OCI_BATCH_ERRORS so that everything valid in the batch is inserted, and rejected values can be discarded. --- plugins/omoracle/omoracle.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index d1b3b955..331b7dd4 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -342,13 +342,12 @@ static int insert_to_db(instanceData* pData) OCIStmtExecute(pData->service, pData->statement, pData->error, - pData->batch.n, 0, NULL, NULL, OCI_DEFAULT)); + pData->batch.n, 0, NULL, NULL, + OCI_BATCH_ERRORS)); - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); - - pData->batch.n = 0; finalize_it: + pData->batch.n = 0; + OCITransCommit(pData->service, pData->error, 0); dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? "succeeded" : "did not succeed"); RETiRet; -- cgit From 8e430258fdc9b0577ea8e54dae21cc5942f90104 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 May 2009 17:38:33 +0200 Subject: added capability to draw configuration graphs - added $GenerateConfigGraph configuration command which can be used to generate nice-looking (and very informative) rsyslog configuration graphs. - added $ActionName configuration directive (currently only used for graph generation, but may find other uses) --- ChangeLog | 13 ++- action.c | 11 +- action.h | 1 + doc/Makefile.am | 3 + doc/rsconf1_generateconfiggraph.html | 121 ++++++++++++++++++++++ doc/rsyslog_conf_global.html | 3 + doc/rsyslog_confgraph_complex.conf | 108 +++++++++++++++++++ doc/rsyslog_confgraph_complex.png | Bin 0 -> 143204 bytes doc/rsyslog_confgraph_std.conf | 79 ++++++++++++++ doc/rsyslog_confgraph_std.png | Bin 0 -> 167756 bytes doc/troubleshoot.html | 9 ++ runtime/rsyslog.h | 1 + tools/syslogd.c | 194 ++++++++++++++++++++++++++++++++++- 13 files changed, 530 insertions(+), 13 deletions(-) create mode 100644 doc/rsconf1_generateconfiggraph.html create mode 100644 doc/rsyslog_confgraph_complex.conf create mode 100644 doc/rsyslog_confgraph_complex.png create mode 100644 doc/rsyslog_confgraph_std.conf create mode 100644 doc/rsyslog_confgraph_std.png diff --git a/ChangeLog b/ChangeLog index 98f59e9e..d37b8e0b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,15 +1,20 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- performance enhancemnt: imtcp calls parser no longer on input thread + but rather inside on of the potentially many main msg queue worker + threads (an enhancement scheduled for all input plugins where this is + possible) +- added $GenerateConfigGraph configuration command which can be used + to generate nice-looking (and very informative) rsyslog configuration + graphs. +- added $ActionName configuration directive (currently only used for + graph generation, but may find other uses) - improved doc * added (hopefully) easier to grasp queue explanation - improved testbench * added tests for queue disk-only mode (checks disk queue logic) - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs -- performance enhancemnt: imtcp calls parser no longer on input thread - but rather inside on of the potentially many main msg queue worker - threads (an enhancement scheduled for all input plugins where this is - possible) --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/action.c b/action.c index 03073153..51620fce 100644 --- a/action.c +++ b/action.c @@ -60,6 +60,7 @@ static int glbliActionResumeInterval = 30; int glbliActionResumeRetryCount = 0; /* how often should suspended actions be retried? */ static int bActionRepMsgHasMsg = 0; /* last messsage repeated... has msg fragment in it */ +static uchar *pszActionName; /* short name for the action */ /* main message queue and its configuration parameters */ static queueType_t ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ static int iActionQueueSize = 1000; /* size of the main message queue above */ @@ -163,8 +164,7 @@ actionResetQueueParams(void) glbliActionResumeRetryCount = 0; /* I guess it is smart to reset this one, too */ - if(pszActionQFName != NULL) - d_free(pszActionQFName); + d_free(pszActionQFName); pszActionQFName = NULL; /* prefix for the main message queue file */ RETiRet; @@ -191,8 +191,8 @@ rsRetVal actionDestruct(action_t *pThis) SYNC_OBJ_TOOL_EXIT(pThis); pthread_mutex_destroy(&pThis->mutActExec); - if(pThis->ppTpl != NULL) - d_free(pThis->ppTpl); + d_free(pThis->pszName); + d_free(pThis->ppTpl); d_free(pThis); RETiRet; @@ -829,6 +829,7 @@ actionAddCfSysLineHdrl(void) { DEFiRet; + CHKiRet(regCfSysLineHdlr((uchar *)"actionname", 0, eCmdHdlrGetWord, NULL, &pszActionName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszActionQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesize", 0, eCmdHdlrInt, NULL, &iActionQueueSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iActionQueMaxDiskSpace, NULL)); @@ -881,6 +882,8 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques CHKiRet(actionConstruct(&pAction)); /* create action object first */ pAction->pMod = pMod; pAction->pModData = pModData; + pAction->pszName = pszActionName; + pszActionName = NULL; /* free again! */ pAction->bExecWhenPrevSusp = bActExecWhenPrevSusp; pAction->iSecsExecOnceInterval = iActExecOnceInterval; pAction->iExecEveryNthOccur = iActExecEveryNthOccur; diff --git a/action.h b/action.h index f2706af6..2a1487a5 100644 --- a/action.h +++ b/action.h @@ -72,6 +72,7 @@ struct action_s { */ qqueue_t *pQueue; /* action queue */ SYNC_OBJ_TOOL; /* required for mutex support */ + uchar *pszName; /* action name (for documentation) */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ }; typedef struct action_s action_t; diff --git a/doc/Makefile.am b/doc/Makefile.am index 4d9d94ff..0703b8fc 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -78,6 +78,7 @@ html_files = \ rsconf1_filecreatemode.html \ rsconf1_filegroup.html \ rsconf1_fileowner.html \ + rsconf1_generateconfiggraph.html \ rsconf1_gssforwardservicename.html \ rsconf1_gsslistenservicename.html \ rsconf1_gssmode.html \ @@ -113,6 +114,8 @@ html_files = \ src/classes.dia grfx_files = \ + rsyslog_confgraph_complex.png\ + rsyslog_confgraph_std.png \ direct_queue0.png \ direct_queue1.png \ direct_queue2.png \ diff --git a/doc/rsconf1_generateconfiggraph.html b/doc/rsconf1_generateconfiggraph.html new file mode 100644 index 00000000..0b18463a --- /dev/null +++ b/doc/rsconf1_generateconfiggraph.html @@ -0,0 +1,121 @@ + + +rsyslog.conf file + + +back + +

    $GenerateConfigGraph

    +

    Type: global configuration directive

    +

    Default:

    +

    Available Since: 4.3.1

    +

    Description:

    +

    This directive permits to create (hopefully) good-looking visualizations of rsyslogd's +configuration. It does not affect rsyslog operation. If the directive is specified multiple +times, all but the last are ignored. If it is specified, a graph is created. This happens +both during a regular startup as well a config check run. It is recommended to include +this directive only for documentation purposes and remove it from a production +configuraton. +

    The graph is not drawn by rsyslog itself. Instead, it uses the great open source tool +Graphviz to do the actual drawing. This has at least +two advantages: +

      +
    • the graph drawing support code in rsyslog is extremly slim and without overhead +
    • the user may change or further annotate the generated file, thus potentially +improving his documentation +
    +The drawback, of course, is that you need to run Graphviz once you have generated +the control file with rsyslog. Fortunately, the process to do so is rather easy: +
      +
    1. add "$GenerateConfigGraph /path/to/file.dot" to rsyslog.conf (from now on, I +will call the file just file.dot). Optionally, add "$ActionName" statement +in front of those actions that you like to use friendly names with. If you do +this, keep the names short. +
    2. run rsyslog at least once (either in regular or configuration check mode) +
    3. remember to remove the $GenerateConfigGraph directive when you no longer need it (or +comment it out) +
    4. change your working directory to where you place the dot file +
    5. if you would like to edit the rsyslog-generated file, now is the time to do so +
    6. do "dot -Tpng file.dot > file.png" +
    7. remember that you can use "convert -resize 50% file.png resized.png" if +dot's output is too large (likely) or too small. Resizing can be especially useful if +you intend to get a rough overview over your configuration. +
    +After completing these steps, you should have a nice graph of your configuration. Details +are missing, but that is exactly the point. At the start of the graph is always (at least +in this version, could be improved) a node called "inputs" in a tripple hexagon +shape. This represents all inputs active in the system (assuming you have defined some, +what the current version does not check). Next comes the main queue. It is given in a +hexagon shape. That shape indicates that a queue is peresent and used to de-couple +the inbound from the outbound part of the graph. In technical terms, here is a +threading boundary. Action with "real" queues (other than in direct mode) +also utilize this shape. For actions, notice that a "hexagon action" creates +a deep copy of the message. As such, a "discard hexagon action" actually does +nothing, because it duplicates the message and then discards the duplicate. +At the end of the diagram, you always see a "discard" action. This indicates +that rsyslog discards messages which have been run through all available rules. +

    Edges are labeled with information about when they are taken. For filters, the type of +filter, but not any specifics, are given. It is also indicated if no filter is +applied in the configuration file (by using a "*.*" selector). Edges without +labels are unconditionally taken. The actions themselfs are labeled with the name of +the output module that handles them. If provided, the name given via +"ActionName" is used instead. No further details are provided. +

    If there is anything in red, this should draw your attention. In this case, rsyslogd +has detected something that does not look quite right. A typical example is a discard +action which is followed by some other actions in an action unit. Even though something +may be red, it can be valid - rsyslogd's graph generator does not yet check each and +every speciality, so the configuration may just cover a very uncommon case. +

    Now let's look at some examples. The graph below was generated on a fairly standard +Fedora rsyslog.conf file. It had only the usually commented-out last forwarding action +activated: +

    +rsyslog configuration graph for a default fedora rsyslog.conf +

    This is the typical structure for a simple rsyslog configuration. There are a couple of +actions, each guarded by a filter. Messages run from top to bottom and control branches +whenever a filter evaluates to true. As there is no discard action, all messages will +run through all filters and discarded in the system default discard action right after +all configured actions. +

    +

    A more complex example can be seen in the next graph. This is a configuration I +created for testing the graph-creation features, so it contains a little bit of +everything. However, real-world configurations can look quite complex, too (and I +wouldn't say this one is very complex): +

    + +

    +

    Here, we have a user-defined discard action. You can immediately see this because +processing branches after the first "builtin-file" action. Those messages +where the filter evaluates to true for will never run through the left-hand action +branch. However, there is also a configuration error present: there are two more +actions (now shown red) after the discard action. As the message is discarded, these will +never be executed. Note that the discard branch contains no further filters. This is +because these actions are all part of the same action unit, which is guarded only by +an entry filter. The same is present a bit further down at the node labeled +"write_system_log_2". This note has one more special feature, that is label +was set via "ActionName", thus is does not have standard form (the same +happened to the node named "Forward" right at the top of the diagram. +Inside this diagram, the "Forward" node is executed asynchonously on its own +queue. All others are executed synchronously. +

    Configuration graphs are useful for documenting a setup, but are also a great +troubleshooting resource. It is important to +remember that these graphs are generated +from rsyslogd's in-memory action processing structures. You can not get closer +to understanding on how rsyslog interpreted its configuration files. +So if the graph does not look +what you intended to do, there is probably something worng in rsyslog.conf. +

    If something is not working as expected, but you do not spot the error immediately, +I recommend to generate a graph and zoom it so that you see all of it in one great picture. +You may not be able to read anything, but the structure should look good to you and +so you can zoom into those areas that draw your attention. +

    Sample:

    +

    $DirOwner /path/to/graphfile-file.dot

    + +

    [rsyslog.conf overview] [manual +index] [rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + + diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 3e33f0da..43eacc43 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -18,6 +18,8 @@ many parameter settings modify queue parameters. If in doubt, use the default, it is usually well-chosen and applicable in most cases.

    • $ActionExecOnlyWhenPreviousIsSuspended
      • +
    • $ActionName <a_single_word> - used primarily for documentation, e.g. when +generating a configuration graph. Available sice 4.3.1.
    • $ActionExecOnlyOnceEveryInterval <seconds> - execute action only if the last execute is at last <seconds> seconds in the past (more info in ommail, @@ -116,6 +118,7 @@ default 60000 (1 minute)]
    • $FileCreateMode
    • $FileGroup
    • $FileOwner
      • +
    • $GenerateConfigGraph
    • $GssForwardServiceName
    • $GssListenServiceName
    • $GssMode
      • diff --git a/doc/rsyslog_confgraph_complex.conf b/doc/rsyslog_confgraph_complex.conf new file mode 100644 index 00000000..3d7ec0a3 --- /dev/null +++ b/doc/rsyslog_confgraph_complex.conf @@ -0,0 +1,108 @@ +$DebugPrintTemplateList off +$DebugPrintCfSysLineHandlerList off +$DebugPrintModuleList off +#$ResetConfigVariables +$ErrorMessagesToStderr off +$ModLoad /home/rger/proj/rsyslog/plugins/imuxsock/.libs/imuxsock.so +#$ModLoad /home/rger/proj/rsyslog/plugins/imklog/.libs/imklog +#$ModLoad /home/rger/proj/rsyslog/plugins/imtcp/.libs/imtcp +$ModLoad /home/rger/proj/rsyslog/plugins/imtcp/.libs/imtcp +$ModLoad /home/rger/proj/rsyslog/plugins/imudp/.libs/imudp +$ModLoad /home/rger/proj/rsyslog/plugins/omstdout/.libs/omstdout +$ModLoad /home/rger/proj/rsyslog/plugins/omprog/.libs/omprog +$ModLoad /home/rger/proj/rsyslog/plugins/omtesting/.libs/omtesting +#$ModLoad /home/rger/proj/rsyslog/plugins/ommail/.libs/ommail +# +# +# PGSQL testing +$ModLoad /home/rger/proj/rsyslog/plugins/ompgsql/.libs/ompgsql.so +$template pgfmt,"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%');",STDSQL +#$ActionQueueType linkedlist +#*.* :ompgsql:127.0.0.1,rsyslog,postgres,;pgfmt + +#$ActionOMStdoutArrayInterface on +#*.* :omstdout: + +$ActionResumeInterval 4 +$ActionResumeRetryCount 3 +$ActionQueueType LinkedList # run asynchronously +$ActionName Forward to 172.19.3.9 +*.* @@172.19.3.9:10514 +#*.* :omtesting:randfail +#*.* :omtesting:always_suspend +#*.* :omtesting:fail 2 2 + +#$UDPServerTimeRequery 10 +$UDPServerRun 514 +$inputtcpmaxsessions 2000 +$InputTCPServerRun 12514 + +#$PrivDropToUser rger +#$InputTCPServerInputName tcp/514 +#$InputTCPServerAddtlFrameDelimiter 10 +#$InputTCPServerRun 514 +#$AllowedSender UDP,127.0.0.1/32 +#$AllowedSender TCP,127.0.0.1/32 + +$PreserveFQDN off + +#$HUPisRestart on + +#$MainMsgQueueType direct +$MainMsgQueueType linkedlist +$MainMsgQueueDequeueBatchSize 200 +#$MainMsgQueueWorkerTimeoutThreadShutdown -1 + +#---- test DA mode +# set spool locations and switch queue to disk assisted mode +$WorkDirectory spool +$MainMsgQueueSize 200 # this *should* trigger moving on to DA mode... +# note: we must set QueueSize sufficiently high, so that 70% (light delay mark) +# is high enough above HighWatermark! +$MainMsgQueueHighWatermark 80 +$MainMsgQueueLowWatermark 40 +$MainMsgQueueFilename mainq +$MainMsgQueueType linkedlist +# ucomment, as we now have an issue (finally the test case works ;)) +#$MainMsgQueueDequeueBatchSize 80 +#---- end test DA mode + +#$template test,"%timereported:::date-rfc3339%,%timereported:::date-mysql%,%timereported:::date-subseconds%, %timegenerated:::date-mysql%, %timegenerated:::date-subseconds%, msg: %msg%\n" +#$template db,"re: '%msg:R,ERE,1,FIELD:dsn=([0-9]+\.[0-9]+\.[0-9])--end%', msg: '%msg%'\n" +#$template db,"re: '%msg:R,ERE,1,ZERO:dsn=([0-9]+\.[0-9]+\.[0-9])--end%', msg: '%msg%'\n" +#$template DEBUG,"Debug line with all properties:\nFROMHOST: '%FROMHOST%', fromhost-ip: '%fromhost-ip%, HOSTNAME: '%HOSTNAME%', PRI: %PRI%,\nsyslogtag '%syslogtag%', programname: '%programname%', APP-NAME: '%APP-NAME%', PROCID: '%PROCID%', MSGID: '%MSGID%',\nTIMESTAMP: '%TIMESTAMP%', STRUCTURED-DATA: '%STRUCTURED-DATA%',\nmsg: '%msg%'\nescaped msg: '%msg:::drop-cc%'\nrawmsg: '%rawmsg%'\n\n" +$template csv,"%syslogtag:::csv%,%msg:::upppercase,csv%,%msg%\n" +*.* -/home/rger/proj/rsyslog/logfile +kern.* -/home/rger/proj/rsyslog/logfile +$ActionExecOnlyWhenPreviousIsSuspended on +& -/tmp/xyz/uuu +$ActionExecOnlyWhenPreviousIsSuspended off +& ~ +& -/tmp/xyz/uuu2 +& -/tmp/xyz/uuu3 + + +#$template dynfile,"/home/rger/proj/rsyslog/test-%syslogtag%" +#*.* -?dynfile +#:msg, ereregex, "test|tast" /home/rger/proj/rsyslog/ere +#if strlen($syslogtag & strlen($msg)) > 10 then /home/rger/proj/rsyslog/longlog +#if strlen($msg) > 10 then /home/rger/proj/rsyslog/longlog +#if tolower($msg) contains 'test' then /home/rger/proj/rsyslog/longlog +#if $msg contains 'test' then /home/rger/proj/rsyslog/longlog + +#$ActionOMProgBinary /home/rger/proj/rsyslog/consumer +#*.* :omprog: + +#$actionresumeretryCount -1 +#$actionResumeInterval 4 +#$template dynfile,"/mnt2/logs/logfile.log" +#*.* /mnt2/logs/logfile.log +#if $msg contains 'test' then ?dynfile +#*.* ?dynfile +:msg, contains, "test " /tmpo/sdafsdf + +$ActionName write_system_log_2 +if $msg == 'test' then /tmpo/sdafsdf2 +& /tmpo/234234 +*.* @@(o,z9)172.19.3.21:10514 +$GenerateConfigGraph /home/rger/proj/rsyslog/rsyslog.dot diff --git a/doc/rsyslog_confgraph_complex.png b/doc/rsyslog_confgraph_complex.png new file mode 100644 index 00000000..21c04c57 Binary files /dev/null and b/doc/rsyslog_confgraph_complex.png differ diff --git a/doc/rsyslog_confgraph_std.conf b/doc/rsyslog_confgraph_std.conf new file mode 100644 index 00000000..64c9a18a --- /dev/null +++ b/doc/rsyslog_confgraph_std.conf @@ -0,0 +1,79 @@ +#rsyslog v3 config file + +# if you experience problems, check +# http://www.rsyslog.com/troubleshoot for assistance + +#### MODULES #### + +$ModLoad imuxsock.so # provides support for local system logging (e.g. via logger command) +$ModLoad imklog.so # provides kernel logging support (previously done by rklogd) +#$ModLoad immark.so # provides --MARK-- message capability + +# Provides UDP syslog reception +#$ModLoad imudp.so +#$UDPServerRun 514 + +# Provides TCP syslog reception +#$ModLoad imtcp.so +#$InputTCPServerRun 514 + + +#### GLOBAL DIRECTIVES #### + +# Use default timestamp format +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# File syncing capability is disabled by default. This feature is usually not required, +# not useful and an extreme performance hit +#$ActionFileEnableSync on + + +#### RULES #### + +# Log all kernel messages to the console. +# Logging much else clutters up the screen. +#kern.* /dev/console + +# Log anything (except mail) of level info or higher. +# Don't log private authentication messages! +*.info;mail.none;authpriv.none;cron.none /var/log/messages + +# The authpriv file has restricted access. +authpriv.* /var/log/secure + +# Log all the mail messages in one place. +mail.* -/var/log/maillog + + +# Log cron stuff +cron.* /var/log/cron + +# Everybody gets emergency messages +*.emerg * + +# Save news errors of level crit and higher in a special file. +uucp,news.crit /var/log/spooler + +# Save boot messages also to boot.log +local7.* /var/log/boot.log + + + +# ### begin forwarding rule ### +# The statement between the begin ... end define a SINGLE forwarding +# rule. They belong together, do NOT split them. If you create multiple +# forwarding rules, duplicate the whole block! +# Remote Logging (we use TCP for reliable delivery) +# +# An on-disk queue is created for this action. If the remote host is +# down, messages are spooled to disk and sent when it is up again. +#$WorkDirectory /var/spppl/rsyslog # where to place spool files +#$ActionQueueFileName fwdRule1 # unique name prefix for spool files +#$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) +#$ActionQueueSaveOnShutdown on # save messages to disk on shutdown +$ActionQueueType LinkedList # run asynchronously +#$ActionResumeRetryCount -1 # infinite retries if host is down +# remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional +*.* @@remote-host:514 +# ### end of the forwarding rule ### +$GenerateConfigGraph /home/rger/proj/rsyslog/rsyslog.dot diff --git a/doc/rsyslog_confgraph_std.png b/doc/rsyslog_confgraph_std.png new file mode 100644 index 00000000..655a7f82 Binary files /dev/null and b/doc/rsyslog_confgraph_std.png differ diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index e655c2ef..cb4367f6 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -28,6 +28,15 @@ mode can be used in parallel to a running instance of rsyslogd.

      /path/to/rsyslogd -f/path/to/config-file -N1

      You should also specify other options you usually give (like -c3 and whatever else). Any problems experienced are reported to stderr [aka "your screen" (if not redirected)]. +

      Configuration Graphs +

      Starting with rsyslog 4.3.1, the +"$GenerateConfigGraph" +command is supported, a very valuable troubleshooting tool. It permits to +generate a graph of how rsyslogd understood its configuration file. It is assumed that +many configuration issues can easily be detected just by looking at the configuration graph. +Full details of how to generate the graphs, and what to look for can be found in the +"$GenerateConfigGraph" +manual page.

      Asking for Help

      If you can't find the answer yourself, you should look at these places for community help. diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 026fbbed..fd5a5371 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -267,6 +267,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ + RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 8c86c12e..3a751a30 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -284,6 +284,7 @@ static int gidDropPriv = 0; /* group-id to which priveleges should be dropped to extern int errno; +static uchar *pszConfDAGFile = NULL; /* name of config DAG file, non-NULL means generate one */ /* main message queue and its configuration parameters */ static qqueue_t *pMsgQueue = NULL; /* the main message queue */ static int iMainMsgQueueSize = 10000; /* size of the main message queue above */ @@ -1939,10 +1940,9 @@ static void doDie(int sig) static void freeAllDynMemForTermination(void) { - if(pszMainMsgQFName != NULL) - free(pszMainMsgQFName); - if(pModDir != NULL) - free(pModDir); + free(pszMainMsgQFName); + free(pModDir); + free(pszConfDAGFile); } @@ -2210,6 +2210,184 @@ static void freeSelectors(void) } +/* helper to generateConfigDAG, to print out all actions via + * the llExecFunc() facility. + * rgerhards, 2007-08-02 + */ +struct dag_info { + FILE *fp; /* output file */ + int iActUnit; /* current action unit number */ + int iAct; /* current action in unit */ + int bDiscarded; /* message discarded (config error) */ + }; +DEFFUNC_llExecFunc(generateConfigDAGAction) +{ + action_t *pAction; + uchar *pszModName; + uchar *pszVertexName; + struct dag_info *pDagInfo; + DEFiRet; + + pDagInfo = (struct dag_info*) pParam; + pAction = (action_t*) pData; + + pszModName = module.GetStateName(pAction->pMod); + + /* vertex */ + if(pAction->pszName == NULL) { + if(!strcmp((char*)pszModName, "builtin-discard")) + pszVertexName = (uchar*)"discard"; + else + pszVertexName = pszModName; + } else { + pszVertexName = pAction->pszName; + } + + fprintf(pDagInfo->fp, "\tact%d_%d\t\t[label=\"%s\"%s%s]\n", + pDagInfo->iActUnit, pDagInfo->iAct, pszVertexName, + pDagInfo->bDiscarded ? " style=dotted color=red" : "", + (pAction->pQueue->qType == QUEUETYPE_DIRECT) ? "" : " shape=hexagon" + ); + + /* edge */ + if(pDagInfo->iAct == 0) { + } else { + fprintf(pDagInfo->fp, "\tact%d_%d -> act%d_%d[%s%s]\n", + pDagInfo->iActUnit, pDagInfo->iAct - 1, + pDagInfo->iActUnit, pDagInfo->iAct, + pDagInfo->bDiscarded ? " style=dotted color=red" : "", + pAction->bExecWhenPrevSusp ? " label=\"only if\\nsuspended\"" : "" ); + } + + /* check for discard */ + if(!strcmp((char*) pszModName, "builtin-discard")) { + fprintf(pDagInfo->fp, "\tact%d_%d\t\t[shape=box]\n", + pDagInfo->iActUnit, pDagInfo->iAct); + pDagInfo->bDiscarded = 1; + } + + + ++pDagInfo->iAct; + + RETiRet; +} + + +/* create config DAG + * This functions takes a rsyslog config and produces a .dot file for use + * with graphviz (http://www.graphviz.org). This is done in an effort to + * document, and also potentially troubleshoot, configurations. Plus, I + * consider it a nice feature to explain some concepts. Note that the + * current version only produces a graph with relatively little information. + * This is a foundation that may be later expanded (if it turns out to be + * useful enough). + * rgerhards, 2009-05-11 + */ +static rsRetVal +generateConfigDAG(uchar *pszDAGFile) +{ + selector_t *f; + FILE *fp; + int iActUnit = 1; + int bHasFilter = 0; /* filter associated with this action unit? */ + int bHadFilter; + int i; + struct dag_info dagInfo; + char *pszFilterName; + char szConnectingNode[64]; + DEFiRet; + + assert(pszDAGFile != NULL); + + if((fp = fopen((char*) pszDAGFile, "w")) == NULL) { + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*) + "configuraton graph output file could not be opened, none generated", 0); + ABORT_FINALIZE(RS_RET_FILENAME_INVALID); + } + + dagInfo.fp = fp; + + /* from here on, we assume writes go well. This here is a really + * unimportant utility function and if something goes wrong, it has + * almost no effect. So let's not overdo this... + */ + fprintf(fp, "# graph created by rsyslog " VERSION "\n\n" + "# use the dot tool from http://www.graphviz.org to visualize!\n" + "digraph rsyslogConfig {\n" + "\tinputs [shape=tripleoctagon]\n" + "\tinputs -> act0_0\n" + "\tact0_0 [label=\"main\\nqueue\" shape=hexagon]\n" + /*"\tmainq -> act1_0\n"*/ + ); + strcpy(szConnectingNode, "act0_0"); + dagInfo.bDiscarded = 0; + + for(f = Files; f != NULL ; f = f->f_next) { + /* BSD-Style filters are currently ignored */ + bHadFilter = bHasFilter; + if(f->f_filter_type == FILTER_PRI) { + bHasFilter = 0; + for (i = 0; i <= LOG_NFACILITIES; i++) + if (f->f_filterData.f_pmask[i] != 0xff) { + bHasFilter = 1; + break; + } + } else { + bHasFilter = 1; + } + + /* we know we have a filter, so it can be false */ + switch(f->f_filter_type) { + case FILTER_PRI: + pszFilterName = "pri filter"; + break; + case FILTER_PROP: + pszFilterName = "property filter"; + break; + case FILTER_EXPR: + pszFilterName = "script filter"; + break; + } + + /* write action unit node */ + if(bHasFilter) { + fprintf(fp, "\t%s -> act%d_end\t[label=\"%s:\\nfalse\"]\n", + szConnectingNode, iActUnit, pszFilterName); + fprintf(fp, "\t%s -> act%d_0\t[label=\"%s:\\ntrue\"]\n", + szConnectingNode, iActUnit, pszFilterName); + fprintf(fp, "\tact%d_end\t\t\t\t[shape=point]\n", iActUnit); + snprintf(szConnectingNode, sizeof(szConnectingNode), "act%d_end", iActUnit); + } else { + fprintf(fp, "\t%s -> act%d_0\t[label=\"no filter\"]\n", + szConnectingNode, iActUnit); + snprintf(szConnectingNode, sizeof(szConnectingNode), "act%d_0", iActUnit); + } + + /* draw individual nodes */ + dagInfo.iActUnit = iActUnit; + dagInfo.iAct = 0; + dagInfo.bDiscarded = 0; + llExecFunc(&f->llActList, generateConfigDAGAction, &dagInfo); /* actions */ + + /* finish up */ + if(bHasFilter && !dagInfo.bDiscarded) { + fprintf(fp, "\tact%d_%d -> %s\n", + iActUnit, dagInfo.iAct - 1, szConnectingNode); + } + + ++iActUnit; + } + + fprintf(fp, "\t%s -> act%d_0\n", szConnectingNode, iActUnit); + fprintf(fp, "\tact%d_0\t\t[label=discard shape=box]\n" + "}\n", iActUnit); + fclose(fp); + +finalize_it: + RETiRet; +} + + /* helper to dbPrintInitInfo, to print out all actions via * the llExecFunc() facility. * rgerhards, 2007-08-02 @@ -2223,6 +2401,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) RETiRet; } + /* print debug information as part of init(). This pretty much * outputs the whole config of rsyslogd. I've moved this code * out of init() to clean it somewhat up. @@ -2230,7 +2409,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) */ static void dbgPrintInitInfo(void) { - register selector_t *f; + selector_t *f; int iSelNbr = 1; int i; @@ -2467,6 +2646,10 @@ init(void) } } + /* check if we need to generate a config DAG and, if so, do that */ + if(pszConfDAGFile != NULL) + generateConfigDAG(pszConfDAGFile); + /* we are done checking the config - now validate if we should actually run or not. * If not, terminate. -- rgerhards, 2008-07-25 */ @@ -2903,6 +3086,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"debugprintcfsyslinehandlerlist", 0, eCmdHdlrBinary, NULL, &bDebugPrintCfSysLineHandlerList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"moddir", 0, eCmdHdlrGetWord, NULL, &pModDir, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"generateconfiggraph", 0, eCmdHdlrGetWord, NULL, &pszConfDAGFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"errormessagestostderr", 0, eCmdHdlrBinary, NULL, &bErrMsgToStderr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, setMaxMsgSize, NULL, NULL)); -- cgit From d1985c4a65feb5caaea855517431ed564a13ed4c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 May 2009 22:00:48 +0200 Subject: doc: given some concrete advise on the common %hostname% content problem --- doc/troubleshoot.html | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index cb4367f6..a8855fd4 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -18,7 +18,14 @@ is a rsyslog-doc package, that often needs to be installed separately.

      Malformed Messages and Message Properties

      A common trouble source are ill-formed syslog messages, which lead to to all sorts of interesting problems, including malformed hostnames and dates. -Read the quoted guide to find relief. +Read the quoted guide to find relief. A common symptom is that the %HOSTNAME% property is +used for generating dynafile names, but some glibberish shows up. This is caused by the +malformed syslog messages, so be sure to read the +guide if you face that problem. Just let me add that the +common work-around is to use %FROMHOST% or %FROMHOST-IP% instead. These do not take the +hostname from the message, but rather use the host that sent the message (taken from +the socket layer). Of course, this does not work over NAT or relay chains, where the +only cure is to make sure senders emit well-formed messages.

      Configuration Problems

      Rsyslog 3.21.1 and above has been enhanced to support extended configuration checking. It offers a special command line switch (-N1) that puts it into "config verfication mode". -- cgit From 1bbd2b07f8d7279dd844c7cd52c72af5da5abdb8 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Sun, 17 May 2009 21:53:29 +0200 Subject: Add shave support shave tranforms the verbose autotools output into a pretty Kbuild-like one which makes it easier to spot warnings. See also http://damien.lespiau.name/blog/2009/02/18/shave-making-the-autotools-output-sane/ git clone git://git.lespiau.name/shave By default, shave is disabled and you have to explicitly enable it via ./configure --enable-shave. Signed-off-by: Rainer Gerhards --- .gitignore | 2 + configure.ac | 3 ++ m4/shave.m4 | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ shave-libtool.in | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ shave.in | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 315 insertions(+) create mode 100644 m4/shave.m4 create mode 100644 shave-libtool.in create mode 100644 shave.in diff --git a/.gitignore b/.gitignore index ea044fbe..bc362e7f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ log logfile debug core.* +shave +shave-libtool diff --git a/configure.ac b/configure.ac index 6daf3f5a..db4de163 100644 --- a/configure.ac +++ b/configure.ac @@ -699,6 +699,7 @@ AC_ARG_ENABLE(imtemplate, AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to serach for imtemplate to find everything! +SHAVE_INIT # settings for the omprog output module AC_ARG_ENABLE(omprog, @@ -754,6 +755,8 @@ AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) AC_CONFIG_FILES([Makefile \ + shave \ + shave-libtool \ runtime/Makefile \ tools/Makefile \ doc/Makefile \ diff --git a/m4/shave.m4 b/m4/shave.m4 new file mode 100644 index 00000000..e647e579 --- /dev/null +++ b/m4/shave.m4 @@ -0,0 +1,99 @@ +dnl Make automake/libtool output more friendly to humans +dnl +dnl Copyright (c) 2009, Damien Lespiau +dnl +dnl Permission is hereby granted, free of charge, to any person +dnl obtaining a copy of this software and associated documentation +dnl files (the "Software"), to deal in the Software without +dnl restriction, including without limitation the rights to use, +dnl copy, modify, merge, publish, distribute, sublicense, and/or sell +dnl copies of the Software, and to permit persons to whom the +dnl Software is furnished to do so, subject to the following +dnl conditions: +dnl +dnl The above copyright notice and this permission notice shall be +dnl included in all copies or substantial portions of the Software. +dnl +dnl THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +dnl EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +dnl OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +dnl NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +dnl HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +dnl WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +dnl FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +dnl OTHER DEALINGS IN THE SOFTWARE. +dnl +dnl SHAVE_INIT([shavedir],[default_mode]) +dnl +dnl shavedir: the directory where the shave scripts are, it defaults to +dnl $(top_builddir) +dnl default_mode: (enable|disable) default shave mode. This parameter +dnl controls shave's behaviour when no option has been +dnl given to configure. It defaults to disable. +dnl +dnl * SHAVE_INIT should be called late in your configure.(ac|in) file (just +dnl before AC_CONFIG_FILE/AC_OUTPUT is perfect. This macro rewrites CC and +dnl LIBTOOL, you don't want the configure tests to have these variables +dnl re-defined. +dnl * This macro requires GNU make's -s option. + +AC_DEFUN([_SHAVE_ARG_ENABLE], +[ + AC_ARG_ENABLE([shave], + AS_HELP_STRING( + [--enable-shave], + [use shave to make the build pretty [[default=$1]]]),, + [enable_shave=$1] + ) +]) + +AC_DEFUN([SHAVE_INIT], +[ + dnl you can tweak the default value of enable_shave + m4_if([$2], [enable], [_SHAVE_ARG_ENABLE(yes)], [_SHAVE_ARG_ENABLE(no)]) + + if test x"$enable_shave" = xyes; then + dnl where can we find the shave scripts? + m4_if([$1],, + [shavedir="$ac_pwd"], + [shavedir="$ac_pwd/$1"]) + AC_SUBST(shavedir) + + dnl make is now quiet + AC_SUBST([MAKEFLAGS], [-s]) + AC_SUBST([AM_MAKEFLAGS], ['`test -z $V && echo -s`']) + + dnl we need sed + AC_CHECK_PROG(SED,sed,sed,false) + + dnl substitute libtool + SHAVE_SAVED_LIBTOOL=$LIBTOOL + LIBTOOL="${SHELL} ${shavedir}/shave-libtool '${SHAVE_SAVED_LIBTOOL}'" + AC_SUBST(LIBTOOL) + + dnl substitute cc/cxx + SHAVE_SAVED_CC=$CC + SHAVE_SAVED_CXX=$CXX + SHAVE_SAVED_FC=$FC + SHAVE_SAVED_F77=$F77 + SHAVE_SAVED_OBJC=$OBJC + CC="${SHELL} ${shavedir}/shave cc ${SHAVE_SAVED_CC}" + CXX="${SHELL} ${shavedir}/shave cxx ${SHAVE_SAVED_CXX}" + FC="${SHELL} ${shavedir}/shave fc ${SHAVE_SAVED_FC}" + F77="${SHELL} ${shavedir}/shave f77 ${SHAVE_SAVED_F77}" + OBJC="${SHELL} ${shavedir}/shave objc ${SHAVE_SAVED_OBJC}" + AC_SUBST(CC) + AC_SUBST(CXX) + AC_SUBST(FC) + AC_SUBST(F77) + AC_SUBST(OBJC) + + V=@ + else + V=1 + fi + Q='$(V:1=)' + AC_SUBST(V) + AC_SUBST(Q) +]) + diff --git a/shave-libtool.in b/shave-libtool.in new file mode 100644 index 00000000..54ebd690 --- /dev/null +++ b/shave-libtool.in @@ -0,0 +1,109 @@ +#!/bin/sh +# +# Copyright (c) 2009, Damien Lespiau +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# we need sed +SED=@SED@ +if test -z "$SED" ; then +SED=sed +fi + +lt_unmangle () +{ + last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` +} + +# the real libtool to use +LIBTOOL="$1" +shift + +# if 1, don't print anything, the underlaying wrapper will do it +pass_though=0 + +# scan the arguments, keep the right ones for libtool, and discover the mode +preserved_args= + +# have we seen the --tag option of libtool in the command line ? +tag_seen=0 + +while test "$#" -gt 0; do + opt="$1" + shift + + case $opt in + --mode=*) + mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` + preserved_args="$preserved_args $opt" + ;; + -o) + lt_output="$1" + preserved_args="$preserved_args $opt" + ;; + --tag=*) + tag_seen=1 + preserved_args="$preserved_args $opt" + ;; + *) + preserved_args="$preserved_args $opt" + ;; + esac +done + +case "$mode" in +compile) + # shave will be called and print the actual CC/CXX/LINK line + preserved_args="$preserved_args --shave-mode=$mode" + pass_though=1 + ;; +link) + preserved_args="$preserved_args --shave-mode=$mode" + Q=" LINK " + ;; +*) + # let's u + # echo "*** libtool: Unimplemented mode: $mode, fill a bug report" + ;; +esac + +lt_unmangle "$lt_output" +output=$last_result + +# automake does not add a --tag switch to its libtool invocation when +# assembling a .s file and rely on libtool to infer the right action based +# on the compiler name. As shave is using CC to hook a wrapper, libtool gets +# confused. Let's detect these cases and add a --tag=CC option. +tag="" +if test $tag_seen -eq 0 -a x"$mode" = xcompile; then + tag="--tag=CC" +fi + +if test -z $V; then + if test $pass_though -eq 0; then + echo "$Q$output" + fi + $LIBTOOL --silent $tag $preserved_args +else + echo $LIBTOOL $tag $preserved_args + $LIBTOOL $tag $preserved_args +fi diff --git a/shave.in b/shave.in new file mode 100644 index 00000000..afed42e1 --- /dev/null +++ b/shave.in @@ -0,0 +1,102 @@ +#!/bin/sh +# +# Copyright (c) 2009, Damien Lespiau +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# we need sed +SED=@SED@ +if test -z "$SED" ; then +SED=sed +fi + +lt_unmangle () +{ + last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` +} + +# the tool to wrap (cc, cxx, ar, ranlib, ..) +tool="$1" +shift + +# the reel tool (to call) +REEL_TOOL="$1" +shift + +pass_through=0 +preserved_args= +while test "$#" -gt 0; do + opt="$1" + shift + + case $opt in + --shave-mode=*) + mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` + ;; + -o) + lt_output="$1" + preserved_args="$preserved_args $opt" + ;; + *) + preserved_args="$preserved_args $opt" + ;; + esac +done + +# mode=link is handled in the libtool wrapper +case "$mode,$tool" in +link,*) + pass_through=1 + ;; +*,cxx) + Q=" CXX " + ;; +*,cc) + Q=" CC " + ;; +*,fc) + Q=" FC " + ;; +*,f77) + Q=" F77 " + ;; +*,objc) + Q=" OBJC " + ;; +*,*) + # should not happen + Q=" CC " + ;; +esac + +lt_unmangle "$lt_output" +output=$last_result + +if test -z $V; then + if test $pass_through -eq 0; then + echo "$Q$output" + fi + $REEL_TOOL $preserved_args +else + echo $REEL_TOOL $preserved_args + $REEL_TOOL $preserved_args +fi -- cgit From 81875845e63bf64922726c3da134e43b56e5d879 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Sun, 17 May 2009 21:55:40 +0200 Subject: Add autogen.sh This file is useful for building rsyslog from git. It is a shortcut for autoreconf and ./configure and enables shave by default. Signed-off-by: Rainer Gerhards --- autogen.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 autogen.sh diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 00000000..daa87a2a --- /dev/null +++ b/autogen.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +(test -f $srcdir/configure.ac) || { + echo -n "**Error**: Directory "\`$srcdir\'" does not look like the" + echo " top-level package directory" + exit 1 +} + +if test -z "$*"; then + echo "**Warning**: I am going to run \`configure' with no arguments." + echo "If you wish to pass any to it, please specify them on the" + echo \`$0\'" command line." + echo +fi + +(cd $srcdir && autoreconf --verbose --force --install) || exit 1 + +conf_flags="--enable-shave --cache-file=config.cache" + +if test x$NOCONFIGURE = x; then + echo Running $srcdir/configure $conf_flags "$@" ... + $srcdir/configure $conf_flags "$@" \ + && echo Now type \`make\' to compile. || exit 1 +else + echo Skipping configure process. +fi + + -- cgit From 735a02c65622da0bc9c2d6b4ce44d126d4c766aa Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 01:13:17 +0200 Subject: Cleanup and typo fixes Signed-off-by: Rainer Gerhards --- configure.ac | 8 ++------ doc/dev_oplugins.html | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index db4de163..a93a968c 100644 --- a/configure.ac +++ b/configure.ac @@ -697,7 +697,7 @@ AC_ARG_ENABLE(imtemplate, # for samples # AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) -# end of copy template - be sure to serach for imtemplate to find everything! +# end of copy template - be sure to search for imtemplate to find everything! SHAVE_INIT @@ -732,7 +732,7 @@ AC_ARG_ENABLE(omtemplate, # for samples # AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) -# end of copy template - be sure to serach for omtemplate to find everything! +# end of copy template - be sure to search for omtemplate to find everything! # settings for omstdout @@ -747,10 +747,6 @@ AC_ARG_ENABLE(omstdout, esac], [enable_omstdout=no] ) -# -# you may want to do some library checks here - see snmp, mysql, pgsql modules -# for samples -# AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html index 5bfc974c..cc2f7f38 100644 --- a/doc/dev_oplugins.html +++ b/doc/dev_oplugins.html @@ -47,7 +47,7 @@ copying omtemplate. Then, the basic steps you need to do are:

    • mv omtemplate.c your-plugin.c
    • cd ../..
    • vi Makefile.am configure.ac -
      serach for omtemplate, copy and modify (follow comments) +
      search for omtemplate, copy and modify (follow comments)

    Basically, this is all you need to do ... Well, except, of course, coding your plugin ;). For testing, you need rsyslog's debugging support. Some useful -- cgit From 5ab3e8005f62c2016bdb2d3b099d1d8c1a4cf2bc Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:07:07 +0200 Subject: Fix compiler warnings include for memcpy and strlen. Signed-off-by: Rainer Gerhards --- runtime/vm.c | 1 + tcps_sess.c | 1 + tests/rscript.c | 1 + 3 files changed, 3 insertions(+) diff --git a/runtime/vm.c b/runtime/vm.c index 41d3e483..125b0d21 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -25,6 +25,7 @@ #include "config.h" #include +#include #include #include diff --git a/tcps_sess.c b/tcps_sess.c index 2a3ec0df..ceb6142f 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -29,6 +29,7 @@ */ #include "config.h" #include +#include #include #include #include diff --git a/tests/rscript.c b/tests/rscript.c index 6b232f5f..ce81491c 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -24,6 +24,7 @@ */ #include "config.h" #include +#include #include #include -- cgit From 460edfe27fcca96e88d6541a3e1ccb49ab4b1a7a Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:41:31 +0200 Subject: Do not fail "make check" if omstdout is not enabled Only run omod-if-array.sh and parsertest.sh test if omstdout plugin is enabled. Remove the comment and fix the help output for --enable-stdout (default was "no", not "yes") Signed-off-by: Rainer Gerhards --- configure.ac | 4 +--- tests/Makefile.am | 9 ++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index a93a968c..9b160379 100644 --- a/configure.ac +++ b/configure.ac @@ -736,10 +736,8 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # settings for omstdout -# note that "make check" fails, if omstdout is not enabled (thus we enable -# it by default). AC_ARG_ENABLE(omstdout, - [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=yes@:>@])], + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], [case "${enableval}" in yes) enable_omstdout="yes" ;; no) enable_omstdout="no" ;; diff --git a/tests/Makefile.am b/tests/Makefile.am index 0f4cbce1..ed415a1e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,9 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh diskqueue.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh +if ENABLE_OMSTDOUT +TESTS += omod-if-array.sh parsertest.sh +endif TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -8,7 +11,7 @@ test_files = testbench.h runtime-dummy.c EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ - cfg2.cfgtest \ + cfg2.cfgtest \ cfg2.testin \ cfg3.cfgtest \ cfg3.testin \ @@ -19,7 +22,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ NoExistFile.cfgtest \ testsuites/parse1.conf \ testsuites/1.parse1 \ - testsuites/rfc3164.parse1 \ + testsuites/rfc3164.parse1 \ testsuites/rfc5424-1.parse1 \ testsuites/rfc5424-2.parse1 \ testsuites/rfc5424-3.parse1 \ -- cgit From 67974ecd5df103b4d69932410ea0147cba469f9b Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:47:59 +0200 Subject: Rename want_gssapi_krb5 to enable_gssapi_krb5 Be consistent and rename the configure variable want_gssapi_krb5 to enable_gssapi_krb5. Signed-off-by: Rainer Gerhards --- configure.ac | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index 9b160379..17c527a8 100644 --- a/configure.ac +++ b/configure.ac @@ -197,13 +197,13 @@ fi AC_ARG_ENABLE(gssapi_krb5, [AS_HELP_STRING([--enable-gssapi-krb5],[Enable GSSAPI Kerberos 5 support @<:@default=no@:>@])], [case "${enableval}" in - yes) want_gssapi_krb5="yes" ;; - no) want_gssapi_krb5="no" ;; + yes) enable_gssapi_krb5="yes" ;; + no) enable_gssapi_krb5="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-gssapi-krb5) ;; esac], - [want_gssapi_krb5=no] + [enable_gssapi_krb5=no] ) -if test $want_gssapi_krb5 = yes; then +if test $enable_gssapi_krb5 = yes; then AC_CHECK_LIB(gssapi_krb5, gss_acquire_cred, [ AC_CHECK_HEADER(gssapi/gssapi.h, [ AC_DEFINE(USE_GSSAPI,, @@ -213,7 +213,7 @@ if test $want_gssapi_krb5 = yes; then ]) ]) fi -AM_CONDITIONAL(ENABLE_GSSAPI, test x$want_gssapi_krb5 = xyes) +AM_CONDITIONAL(ENABLE_GSSAPI, test x$enable_gssapi_krb5 = xyes) # multithreading via pthreads AC_ARG_ENABLE(pthreads, @@ -803,7 +803,7 @@ echo "omstdout module will be compiled: $enable_omstdout" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" echo "GnuTLS network stream driver enabled: $enable_gnutls" -echo "Enable GSSAPI Kerberos 5 support: $want_gssapi_krb5" +echo "Enable GSSAPI Kerberos 5 support: $enable_gssapi_krb5" echo "Debug mode enabled: $enable_debug" echo "Runtime Instrumentation enabled: $enable_rtinst" echo "Diagnostic tools enabled: $enable_diagtools" -- cgit From 541d7ad9401d3a606c9ecc079eb2de875e5d6f0c Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:50:29 +0200 Subject: Improve ./configure output Improve readability of the ./configure output by grouping relevant entries and indenting them. Signed-off-by: Rainer Gerhards --- configure.ac | 66 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/configure.ac b/configure.ac index 17c527a8..a4f0acfa 100644 --- a/configure.ac +++ b/configure.ac @@ -783,31 +783,41 @@ AC_OUTPUT echo "****************************************************" echo "rsyslog will be compiled with the following settings:" echo -echo "Multithreading support enabled: $enable_pthreads" -echo "Klog functionality enabled: $enable_klog ($os_type)" -echo "Regular expressions support enabled: $enable_regexp" -echo "Zlib compression support enabled: $enable_zlib" -echo "MySql support enabled: $enable_mysql" -echo "libdbi support enabled: $enable_libdbi" -echo "PostgreSQL support enabled: $enable_pgsql" -echo "Oracle (OCI) support enabled: $enable_oracle" -echo "SNMP support enabled: $enable_snmp" -echo "Mail support enabled: $enable_mail" -echo "RELP support enabled: $enable_relp" -echo "imdiag enabled: $enable_imdiag" -echo "file input module enabled: $enable_imfile" -echo "input template module will be compiled: $enable_imtemplate" -echo "output template module will be compiled: $enable_omtemplate" -echo "omprog module will be compiled: $enable_omprog" -echo "omstdout module will be compiled: $enable_omstdout" -echo "Large file support enabled: $enable_largefile" -echo "Networking support enabled: $enable_inet" -echo "GnuTLS network stream driver enabled: $enable_gnutls" -echo "Enable GSSAPI Kerberos 5 support: $enable_gssapi_krb5" -echo "Debug mode enabled: $enable_debug" -echo "Runtime Instrumentation enabled: $enable_rtinst" -echo "Diagnostic tools enabled: $enable_diagtools" -echo "valgrind support settings enabled: $enable_valgrind" -echo "rsyslog runtime will be built: $enable_rsyslogrt" -echo "rsyslogd will be built: $enable_rsyslogd" - +echo " Multithreading support enabled: $enable_pthreads" +echo " Large file support enabled: $enable_largefile" +echo " Networking support enabled: $enable_inet" +echo " Regular expressions support enabled: $enable_regexp" +echo " Zlib compression support enabled: $enable_zlib" +echo " rsyslog runtime will be built: $enable_rsyslogrt" +echo " rsyslogd will be built: $enable_rsyslogd" +echo +echo "---{ input plugins }---" +echo " Klog functionality enabled: $enable_klog ($os_type)" +echo " imdiag enabled: $enable_imdiag" +echo " file input module enabled: $enable_imfile" +echo " input template module will be compiled: $enable_imtemplate" +echo +echo "---{ output plugins }---" +echo " Mail support enabled: $enable_mail" +echo " omprog module will be compiled: $enable_omprog" +echo " omstdout module will be compiled: $enable_omstdout" +echo " output template module will be compiled: $enable_omtemplate" +echo +echo "---{ database support }---" +echo " MySql support enabled: $enable_mysql" +echo " libdbi support enabled: $enable_libdbi" +echo " PostgreSQL support enabled: $enable_pgsql" +echo " Oracle (OCI) support enabled: $enable_oracle" +echo +echo "---{ protocol support }---" +echo " GnuTLS network stream driver enabled: $enable_gnutls" +echo " GSSAPI Kerberos 5 support enabled: $enable_gssapi_krb5" +echo " RELP support enabled: $enable_relp" +echo " SNMP support enabled: $enable_snmp" +echo +echo "---{ debugging support }---" +echo " Debug mode enabled: $enable_debug" +echo " Runtime Instrumentation enabled: $enable_rtinst" +echo " Diagnostic tools enabled: $enable_diagtools" +echo " Valgrind support settings enabled: $enable_valgrind" +echo -- cgit From 4f691b1e1dd3ee6e7f26e57aef41a730e7168a40 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 22:00:53 +0200 Subject: Cleanup configure.ac a little for better readability - Use 2 new lines as separator between configure options. - Remove forgotten copy&paste text. - Reorder and move template configure option to the end. Signed-off-by: Rainer Gerhards --- configure.ac | 76 +++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/configure.ac b/configure.ac index a4f0acfa..8c8db8e1 100644 --- a/configure.ac +++ b/configure.ac @@ -144,6 +144,7 @@ AC_ARG_WITH(moddirs, AM_CONDITIONAL(WITH_MODDIRS, test x$moddirs != x) AC_SUBST(moddirs) + # Large file support AC_ARG_ENABLE(largefile, [AS_HELP_STRING([--enable-largefile],[Enable large file support @<:@default=yes@:>@])], @@ -158,6 +159,7 @@ if test "$enable_largefile" = "no"; then AC_DEFINE(NOLARGEFILE, 1, [Defined when large file support is disabled.]) fi + # Regular expressions AC_ARG_ENABLE(regexp, [AS_HELP_STRING([--enable-regexp],[Enable regular expressions support @<:@default=yes@:>@])], @@ -173,6 +175,7 @@ if test "$enable_regexp" = "yes"; then AC_DEFINE(FEATURE_REGEXP, 1, [Regular expressions support enabled.]) fi + # zlib compression AC_ARG_ENABLE(zlib, [AS_HELP_STRING([--enable-zlib],[Enable zlib compression support @<:@default=yes@:>@])], @@ -193,6 +196,7 @@ if test "$enable_zlib" = "yes"; then fi fi + #gssapi AC_ARG_ENABLE(gssapi_krb5, [AS_HELP_STRING([--enable-gssapi-krb5],[Enable GSSAPI Kerberos 5 support @<:@default=no@:>@])], @@ -215,6 +219,7 @@ if test $enable_gssapi_krb5 = yes; then fi AM_CONDITIONAL(ENABLE_GSSAPI, test x$enable_gssapi_krb5 = xyes) + # multithreading via pthreads AC_ARG_ENABLE(pthreads, [AS_HELP_STRING([--enable-pthreads],[Enable multithreading via pthreads @<:@default=yes@:>@])], @@ -255,6 +260,7 @@ if test "x$enable_pthreads" != "xno"; then ) fi + # klog AC_ARG_ENABLE(klog, [AS_HELP_STRING([--enable-klog],[Integrated klog functionality @<:@default=yes@:>@])], @@ -269,6 +275,7 @@ AM_CONDITIONAL(ENABLE_IMKLOG, test x$enable_klog = xyes) AM_CONDITIONAL(ENABLE_IMKLOG_BSD, test x$os_type = xbsd) AM_CONDITIONAL(ENABLE_IMKLOG_LINUX, test x$os_type = xlinux) + # # SYSLOG_UNIXAF # @@ -289,6 +296,7 @@ AC_ARG_ENABLE([unix], AC_DEFINE([SYSLOG_UNIXAF], [1], [Description]) ]) + # inet AC_ARG_ENABLE(inet, [AS_HELP_STRING([--enable-inet],[Enable networking support @<:@default=yes@:>@])], @@ -304,6 +312,7 @@ if test "$enable_inet" = "yes"; then AC_DEFINE(SYSLOG_INET, 1, [network support is integrated.]) fi + # # The following define determines whether the package adheres to the # file system standard. @@ -325,6 +334,7 @@ AC_ARG_ENABLE([fsstnd], AC_DEFINE([FSSTND], [1], [Description]) ]) + # debug AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], @@ -342,6 +352,7 @@ if test "$enable_debug" = "no"; then AC_DEFINE(NDEBUG, 1, [Defined if debug mode is disabled.]) fi + # runtime instrumentation AC_ARG_ENABLE(rtinst, [AS_HELP_STRING([--enable-rtinst],[Enable runtime instrumentation mode @<:@default=no@:>@])], @@ -356,6 +367,7 @@ if test "$enable_rtinst" = "yes"; then AC_DEFINE(RTINST, 1, [Defined if runtime instrumentation mode is enabled.]) fi + # valgrind AC_ARG_ENABLE(valgrind, [AS_HELP_STRING([--enable-valgrind],[Enable valgrind support settings @<:@default=no@:>@])], @@ -452,6 +464,7 @@ AM_CONDITIONAL(ENABLE_PGSQL, test x$enable_pgsql = xyes) AC_SUBST(PGSQL_CFLAGS) AC_SUBST(PGSQL_LIBS) + # oracle (OCI) support AC_ARG_ENABLE(oracle, [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@])], @@ -484,7 +497,8 @@ fi AM_CONDITIONAL(ENABLE_ORACLE, test x$enable_oracle = xyes) AC_SUBST(ORACLE_CFLAGS) AC_SUBST(ORACLE_LIBS) - + + # libdbi support AC_ARG_ENABLE(libdbi, [AS_HELP_STRING([--enable-libdbi],[Enable libdbi database support @<:@default=no@:>@])], @@ -645,6 +659,7 @@ AM_CONDITIONAL(ENABLE_RELP, test x$enable_relp = xyes) AC_SUBST(RELP_CFLAGS) AC_SUBST(RELP_LIBS) + # RFC 3195 support AC_ARG_ENABLE(rfc3195, [AS_HELP_STRING([--enable-rfc3195],[Enable RFC3195 support @<:@default=no@:>@])], @@ -663,7 +678,7 @@ AC_SUBST(LIBLOGGING_CFLAGS) AC_SUBST(LIBLOGGING_LIBS) -# settings for the file input module; +# settings for the file input module AC_ARG_ENABLE(imfile, [AS_HELP_STRING([--enable-imfile],[file input module enabled @<:@default=no@:>@])], [case "${enableval}" in @@ -673,13 +688,35 @@ AC_ARG_ENABLE(imfile, esac], [enable_imfile=no] ) -# -# you may want to do some library checks here - see snmp, mysql, pgsql modules -# for samples -# AM_CONDITIONAL(ENABLE_IMFILE, test x$enable_imfile = xyes) +# settings for the omprog output module +AC_ARG_ENABLE(omprog, + [AS_HELP_STRING([--enable-omprog],[Compiles omprog module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omprog="yes" ;; + no) enable_omprog="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omprog) ;; + esac], + [enable_omprog=no] +) +AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) + + +# settings for omstdout +AC_ARG_ENABLE(omstdout, + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omstdout="yes" ;; + no) enable_omstdout="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omstdout) ;; + esac], + [enable_omstdout=no] +) +AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) + + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -699,21 +736,6 @@ AC_ARG_ENABLE(imtemplate, AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to search for imtemplate to find everything! -SHAVE_INIT - -# settings for the omprog output module -AC_ARG_ENABLE(omprog, - [AS_HELP_STRING([--enable-omprog],[Compiles omprog template module @<:@default=no@:>@])], - [case "${enableval}" in - yes) enable_omprog="yes" ;; - no) enable_omprog="no" ;; - *) AC_MSG_ERROR(bad value ${enableval} for --enable-omprog) ;; - esac], - [enable_omprog=no] -) -AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) -# end of omprog - # settings for the template output module; copy and modify this code # if you intend to add your own module. Be sure to replace omtemplate @@ -735,17 +757,7 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to search for omtemplate to find everything! -# settings for omstdout -AC_ARG_ENABLE(omstdout, - [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], - [case "${enableval}" in - yes) enable_omstdout="yes" ;; - no) enable_omstdout="no" ;; - *) AC_MSG_ERROR(bad value ${enableval} for --enable-omstdout) ;; - esac], - [enable_omstdout=no] -) -AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) +SHAVE_INIT AC_CONFIG_FILES([Makefile \ -- cgit From 55714cdb440b4baa32f88d55316755555ef93607 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 19 May 2009 09:17:13 +0200 Subject: updated ChangeLog with Michael Biebl's changes --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index d37b8e0b..30fb7bff 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,6 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? -- performance enhancemnt: imtcp calls parser no longer on input thread +- performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker threads (an enhancement scheduled for all input plugins where this is possible) @@ -15,6 +15,7 @@ Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? * added tests for queue disk-only mode (checks disk queue logic) - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- build system improvements - thanks to Michael Biebl --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program -- cgit From 2aca3c2d33dd1a19daf44168e90a5f64bd4095b0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 May 2009 17:35:21 +0200 Subject: made imdiag *just* compile & some cleanup imdiag was never finished (not even really begun), but now I need it. I made the few things that are available compile, but more serious work is required. --- plugins/imdiag/imdiag.c | 36 ++++++++++++++++++++---------------- tcpsrv.c | 17 ++++++----------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 3cd2dcf8..90c5d9ee 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -7,7 +7,7 @@ * * File begun on 2008-07-25 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -48,6 +48,7 @@ #include "module-template.h" #include "net.h" #include "netstrm.h" +#include "netstrms.h" #include "errmsg.h" MODULE_TYPE_INPUT @@ -56,12 +57,13 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(net) DEFobjCurrIf(netstrm) +DEFobjCurrIf(netstrms) DEFobjCurrIf(errmsg) /* Module static data */ netstrms_t *pNS; /**< pointer to network stream subsystem */ -netstrm_t **ppLstn[10]; /**< our netstream listners */ -int iLstnMax = 0; /**< max nbr of listeners currently supported */ +netstrm_t *arrLstn[10]; /**< our netstream listners */ +int iLstnMax = 0; /**< max nbr of listeners currently supported */ /* config settings */ @@ -71,16 +73,16 @@ int iLstnMax = 0; /**< max nbr of listeners currently supported */ * invoked from the netstrm class. -- rgerhards, 2008-04-23 */ static rsRetVal -addTcpLstn(void *pUsr, netstrm_t *pLstn) +addTcpLstn(netstrm_t *pLstn) { DEFiRet; ISOBJ_TYPE_assert(pLstn, netstrm); - if(iLstnMax >= sizeof(ppLstn)/sizeof(netstrm_t)) + if((unsigned)iLstnMax >= sizeof(arrLstn)/sizeof(netstrm_t*)) ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); - ppLstn[pThis->iLstnMax] = pLstn; + arrLstn[iLstnMax] = pLstn; ++iLstnMax; finalize_it: @@ -100,15 +102,15 @@ initNetstrm(void) //CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); //CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); // TODO: set driver! - CHKiRet(netstrms.ConstructFinalize(pThis->pNS)); + CHKiRet(netstrms.ConstructFinalize(&pNS)); /* set up listeners */ - CHKiRet(netstrm.LstnInit(pNS, NULL, addTcpLstn, "127.0.0.1", "44514", 1)); + CHKiRet(netstrm.LstnInit(pNS, NULL, addTcpLstn, "127.0.0.1", (uchar*)"44514", 1)); finalize_it: if(iRet != RS_RET_OK) { - if(pThis->pNS != NULL) - netstrms.Destruct(&pThis->pNS); + if(pNS != NULL) + netstrms.Destruct(&pNS); } RETiRet; } @@ -130,11 +132,12 @@ ENDwillRun BEGINafterRun + int i; CODESTARTafterRun /* do cleanup here */ /* finally close our listen streams */ for(i = 0 ; i < iLstnMax ; ++i) { - netstrm.Destruct(ppLstn + i); + netstrm.Destruct(arrLstn[i]); } /* destruct netstream subsystem */ @@ -146,7 +149,8 @@ BEGINmodExit CODESTARTmodExit /* release objects we used */ objRelease(net, LM_NET_FILENAME); - objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(netstrm, DONT_LOAD_LIB); + objRelease(netstrms, LM_NETSTRMS_FILENAME); objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -169,14 +173,14 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr - pOurTcpsrv = NULL; /* request objects we use */ CHKiRet(objUse(net, LM_NET_FILENAME)); - CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); -#if 0 /* register config file handlers */ +#if 0 CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverrun", 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpmaxsessions", 0, eCmdHdlrInt, @@ -187,9 +191,9 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); +#endif CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); -#endif ENDmodInit diff --git a/tcpsrv.c b/tcpsrv.c index b9434398..7af45d3c 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -17,7 +17,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -219,8 +219,7 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) pThis->pSessions = NULL; /* just to make sure... */ } - if(pThis->TCPLstnPort != NULL) - free(pThis->TCPLstnPort); + free(pThis->TCPLstnPort); /* finally close our listen streams */ for(i = 0 ; i < pThis->iLstnMax ; ++i) { @@ -557,12 +556,9 @@ CODESTARTobjDestruct(tcpsrv) if(pThis->pNS != NULL) netstrms.Destruct(&pThis->pNS); - if(pThis->pszDrvrAuthMode != NULL) - free(pThis->pszDrvrAuthMode); - if(pThis->ppLstn != NULL) - free(pThis->ppLstn); - if(pThis->pszInputName != NULL) - free(pThis->pszInputName); + free(pThis->pszDrvrAuthMode); + free(pThis->ppLstn); + free(pThis->pszInputName); ENDobjDestruct(tcpsrv) @@ -683,8 +679,7 @@ SetInputName(tcpsrv_t *pThis, uchar *name) pszName = NULL; else CHKmalloc(pszName = (uchar*)strdup((char*)name)); - if(pThis->pszInputName != NULL) - free(pThis->pszInputName); + free(pThis->pszInputName); pThis->pszInputName = pszName; finalize_it: RETiRet; -- cgit From aba90e82484118f3568ec51c01de5ba845da589a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 May 2009 17:06:52 +0200 Subject: added capability to run multiple tcp listeners (on different ports) Well, actually this and a lot of related things. I improved the testbench so that the new capabilities are automatically tested and also did some general cleanup. The current multiple tcp listener solution will probably receive some further cleanup, too, but looks quite OK so far. I also reviewed the way tcpsrv et all work, in preparation of using this code for imdiag. I need to document the findings, especially as the code is rather complicated "thanks" to the combination of plain tcp and gssapi transport modes. --- ChangeLog | 1 + doc/imtcp.html | 6 +- plugins/imfile/imfile.c | 5 +- plugins/imgssapi/imgssapi.c | 4 +- plugins/imklog/imklog.c | 7 +- plugins/imtcp/imtcp.c | 35 ++++---- plugins/imudp/imudp.c | 5 +- runtime/Makefile.am | 1 + runtime/datetime.c | 12 +-- runtime/datetime.h | 4 +- runtime/msg.c | 104 +++++++++------------ runtime/msg.h | 8 +- runtime/netstrm.c | 4 +- runtime/netstrm.h | 1 + runtime/parser.c | 2 +- runtime/rsyslog.h | 3 + runtime/unicode-helper.h | 54 +++++++++++ tcps_sess.c | 17 +++- tcps_sess.h | 11 ++- tcpsrv.c | 150 +++++++++++++++++++++---------- tcpsrv.h | 16 +++- tests/Makefile.am | 15 +++- tests/fieldtest.sh | 13 +++ tests/imtcp-multiport.sh | 80 +++++++++++++++++ tests/inputname.sh | 20 +++++ tests/killrsyslog.sh | 7 ++ tests/nettester.c | 101 +++++++++++++++------ tests/omod-if-array.sh | 6 +- tests/parsertest.sh | 6 +- tests/tcpflood.c | 2 - tests/testsuites/1.field1 | 3 + tests/testsuites/1.inputname_imtcp_12514 | 3 + tests/testsuites/1.inputname_imtcp_12515 | 3 + tests/testsuites/1.inputname_imtcp_12516 | 3 + tests/testsuites/field1.conf | 8 ++ tests/testsuites/imtcp-multiport.conf | 13 +++ tests/testsuites/inputname_imtcp.conf | 19 ++++ tools/syslogd.c | 53 +++++------ 38 files changed, 584 insertions(+), 221 deletions(-) create mode 100644 runtime/unicode-helper.h create mode 100755 tests/fieldtest.sh create mode 100755 tests/imtcp-multiport.sh create mode 100755 tests/inputname.sh create mode 100755 tests/killrsyslog.sh create mode 100644 tests/testsuites/1.field1 create mode 100644 tests/testsuites/1.inputname_imtcp_12514 create mode 100644 tests/testsuites/1.inputname_imtcp_12515 create mode 100644 tests/testsuites/1.inputname_imtcp_12516 create mode 100644 tests/testsuites/field1.conf create mode 100644 tests/testsuites/imtcp-multiport.conf create mode 100644 tests/testsuites/inputname_imtcp.conf diff --git a/ChangeLog b/ChangeLog index 30fb7bff..b7a0a67a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- added capability to run multiple tcp listeners (on different ports) - performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker threads (an enhancement scheduled for all input plugins where this is diff --git a/doc/imtcp.html b/doc/imtcp.html index 0ee0f96a..9ea7efa1 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -14,9 +14,10 @@ Encryption can be provided by using stunnel (an alternative is the use the imgssapi modul).

    -

    In the future, multiple receivers may be configured by +

    Multiple receivers may be configured by specifying -$InputTCPServerRun multiple times. This is not currently supported. +$InputTCPServerRun multiple times. This is available since version 4.3.1, earlier +versions do NOT support it.

    Configuration Directives:

      @@ -58,7 +59,6 @@ AuthMode and  network stream driver. Permitted Caveats/Known Bugs:
      • module always binds to all interfaces
        • -
      • only a single listener can be bound
      • can not be loaded together with imgssapi (which includes the functionality of imtcp)
      diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index b0211bf6..92fd30c3 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -45,6 +45,7 @@ #include "errmsg.h" #include "glbl.h" #include "datetime.h" +#include "unicode-helper.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -94,11 +95,11 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputName(pMsg, "imfile"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile")); MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); - MsgSetHOSTNAME(pMsg, (char*)glbl.GetLocalHostName()); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, (char*)pInfo->pszTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index b9d7dfe3..d8791880 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -9,7 +9,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -330,7 +330,7 @@ addGSSListener(void __attribute__((unused)) *pVal, uchar *pNewVal) CHKiRet(tcpsrv.SetCBOnSessAccept(pOurTcpsrv, onSessAccept)); CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); - tcpsrv.configureTCPListen(pOurTcpsrv, (char *) pNewVal); + tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); } diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 20bc34ab..ecb6c100 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -53,6 +53,7 @@ #include "datetime.h" #include "imklog.h" #include "glbl.h" +#include "unicode-helper.h" MODULE_TYPE_INPUT @@ -94,14 +95,14 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputName(pMsg, "imklog"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog")); MsgSetRawMsg(pMsg, (char*)msg); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetMSG(pMsg, (char*)msg); - MsgSetRcvFrom(pMsg, (char*)glbl.GetLocalHostName()); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); - MsgSetHOSTNAME(pMsg, (char*)glbl.GetLocalHostName()); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, (char*)pszTag); pMsg->iFacility = LOG_FAC(iFacility); pMsg->iSeverity = LOG_PRI(iSeverity); diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 5a8a62f6..f0efe1ee 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -3,7 +3,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -56,6 +56,7 @@ #include "dirty.h" #include "cfsysline.h" #include "module-template.h" +#include "unicode-helper.h" #include "net.h" #include "netstrm.h" #include "errmsg.h" @@ -91,7 +92,7 @@ static int isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) { - return net.isAllowedSender((uchar*) "TCP", addr, fromHostFQDN); + return net.isAllowedSender(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN); } @@ -169,7 +170,6 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); - CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? (uchar*)"imtcp" : pszInputName)); CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); /* now set optional params, but only if they were actually configured */ if(pszStrmDrvrAuthMode != NULL) { @@ -178,11 +178,14 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa if(pPermPeersRoot != NULL) { CHKiRet(tcpsrv.SetDrvrPermPeers(pOurTcpsrv, pPermPeersRoot)); } - /* most params set, now start listener */ - tcpsrv.configureTCPListen(pOurTcpsrv, (char *) pNewVal); - CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); } +dbgprintf("XXX: try add listen port %s\n", pNewVal); + /* initialized, now add socket */ + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? + UCHAR_CONSTANT("imtcp") : pszInputName)); + tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); + finalize_it: if(iRet != RS_RET_OK) { errmsg.LogError(0, NO_ERRCODE, "error %d trying to add listener", iRet); @@ -199,7 +202,9 @@ CODESTARTrunInput /* TODO: we must be careful to start the listener here. Currently, tcpsrv.c seems to * do that in ConstructFinalize */ + CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); iRet = tcpsrv.Run(pOurTcpsrv); +finalize_it: ENDrunInput @@ -217,7 +222,7 @@ ENDwillRun BEGINafterRun CODESTARTafterRun /* do cleanup here */ - net.clearAllowedSenders((uchar*)"TCP"); + net.clearAllowedSenders(UCHAR_CONSTANT("TCP")); ENDafterRun @@ -277,21 +282,21 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* register config file handlers */ - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverrun", 0, eCmdHdlrGetWord, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverrun"), 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpmaxsessions", 0, eCmdHdlrInt, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpmaxsessions"), 0, eCmdHdlrInt, NULL, &iTCPSessMax, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdrivermode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdrivermode"), 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverauthmode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdriverauthmode"), 0, eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdriverpermittedpeer"), 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserveraddtlframedelimiter", 0, eCmdHdlrInt, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserveraddtlframedelimiter"), 0, eCmdHdlrInt, NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverinputname", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverinputname"), 0, eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index c7e8c1d4..f3448095 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -43,6 +43,7 @@ #include "msg.h" #include "parser.h" #include "datetime.h" +#include "unicode-helper.h" MODULE_TYPE_INPUT @@ -219,11 +220,11 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); pMsg->iLenRawMsg = lenRcvBuf; - MsgSetInputName(pMsg, "imudp"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp")); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; - MsgSetRcvFrom(pMsg, (char*)fromHost); + MsgSetRcvFrom(pMsg, fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); CHKiRet(submitMsg(pMsg)); } diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 2f0a1aa0..8331882f 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -7,6 +7,7 @@ pkglib_LTLIBRARIES = librsyslog_la_SOURCES = \ rsyslog.c \ rsyslog.h \ + unicode_helper.h \ atomic.h \ syslogd-types.h \ module-template.h \ diff --git a/runtime/datetime.c b/runtime/datetime.c index deb66eb4..19e61a0a 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -150,7 +150,7 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) * \retval The number parsed. */ -static int srSLMGParseInt32(char** ppsz) +static int srSLMGParseInt32(uchar** ppsz) { int i; @@ -172,9 +172,9 @@ static int srSLMGParseInt32(char** ppsz) * could be obtained (restriction added 2008-09-16 by rgerhards). */ static rsRetVal -ParseTIMESTAMP3339(struct syslogTime *pTime, char** ppszTS) +ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) { - char *pszTS = *ppszTS; + uchar *pszTS = *ppszTS; /* variables to temporarily hold time information while we parse */ int year; int month; @@ -234,7 +234,7 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, char** ppszTS) /* Now let's see if we have secfrac */ if(*pszTS == '.') { - char *pszStart = ++pszTS; + uchar *pszStart = ++pszTS; secfrac = srSLMGParseInt32(&pszTS); secfracPrecision = (int) (pszTS - pszStart); } else { @@ -307,7 +307,7 @@ finalize_it: * time() call reduction ;). */ static rsRetVal -ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) +ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) { /* variables to temporarily hold time information while we parse */ int month; @@ -317,7 +317,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) int minute; int second; /* end variables to temporarily hold time information while we parse */ - char *pszTS; + uchar *pszTS; DEFiRet; assert(ppszTS != NULL); diff --git a/runtime/datetime.h b/runtime/datetime.h index 0739588d..efb0a0af 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -36,8 +36,8 @@ typedef struct datetime_s { /* interfaces */ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); - rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, char** ppszTS); - rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, char** pszTS); + rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS); + rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS); int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst); int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst, size_t iLenDst); int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); diff --git a/runtime/msg.c b/runtime/msg.c index 9d5f3838..dbc3c779 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -45,6 +45,7 @@ #include "glbl.h" #include "regexp.h" #include "atomic.h" +#include "unicode-helper.h" /* static data */ DEFobjStaticHelpers @@ -341,52 +342,29 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - if(pThis->pszUxTradMsg != NULL) - free(pThis->pszUxTradMsg); - if(pThis->pszRawMsg != NULL) - free(pThis->pszRawMsg); - if(pThis->pszTAG != NULL) - free(pThis->pszTAG); - if(pThis->pszHOSTNAME != NULL) - free(pThis->pszHOSTNAME); - if(pThis->pszInputName != NULL) - free(pThis->pszInputName); - if(pThis->pszRcvFrom != NULL) - free(pThis->pszRcvFrom); - if(pThis->pszRcvFromIP != NULL) - free(pThis->pszRcvFromIP); - if(pThis->pszMSG != NULL) - free(pThis->pszMSG); - if(pThis->pszFacility != NULL) - free(pThis->pszFacility); - if(pThis->pszFacilityStr != NULL) - free(pThis->pszFacilityStr); - if(pThis->pszSeverity != NULL) - free(pThis->pszSeverity); - if(pThis->pszSeverityStr != NULL) - free(pThis->pszSeverityStr); - if(pThis->pszRcvdAt3164 != NULL) - free(pThis->pszRcvdAt3164); - if(pThis->pszRcvdAt3339 != NULL) - free(pThis->pszRcvdAt3339); - if(pThis->pszRcvdAt_SecFrac != NULL) - free(pThis->pszRcvdAt_SecFrac); - if(pThis->pszRcvdAt_MySQL != NULL) - free(pThis->pszRcvdAt_MySQL); - if(pThis->pszRcvdAt_PgSQL != NULL) - free(pThis->pszRcvdAt_PgSQL); - if(pThis->pszTIMESTAMP3164 != NULL) - free(pThis->pszTIMESTAMP3164); - if(pThis->pszTIMESTAMP3339 != NULL) - free(pThis->pszTIMESTAMP3339); - if(pThis->pszTIMESTAMP_SecFrac != NULL) - free(pThis->pszTIMESTAMP_SecFrac); - if(pThis->pszTIMESTAMP_MySQL != NULL) - free(pThis->pszTIMESTAMP_MySQL); - if(pThis->pszTIMESTAMP_PgSQL != NULL) - free(pThis->pszTIMESTAMP_PgSQL); - if(pThis->pszPRI != NULL) - free(pThis->pszPRI); + free(pThis->pszUxTradMsg); + free(pThis->pszRawMsg); + free(pThis->pszTAG); + free(pThis->pszHOSTNAME); + free(pThis->pszInputName); + free(pThis->pszRcvFrom); + free(pThis->pszRcvFromIP); + free(pThis->pszMSG); + free(pThis->pszFacility); + free(pThis->pszFacilityStr); + free(pThis->pszSeverity); + free(pThis->pszSeverityStr); + free(pThis->pszRcvdAt3164); + free(pThis->pszRcvdAt3339); + free(pThis->pszRcvdAt_SecFrac); + free(pThis->pszRcvdAt_MySQL); + free(pThis->pszRcvdAt_PgSQL); + free(pThis->pszTIMESTAMP3164); + free(pThis->pszTIMESTAMP3339); + free(pThis->pszTIMESTAMP_SecFrac); + free(pThis->pszTIMESTAMP_MySQL); + free(pThis->pszTIMESTAMP_PgSQL); + free(pThis->pszPRI); if(pThis->pCSProgName != NULL) rsCStrDestruct(&pThis->pCSProgName); if(pThis->pCSStrucData != NULL) @@ -1306,15 +1284,15 @@ uchar *getInputName(msg_t *pM) } -char *getRcvFrom(msg_t *pM) +uchar *getRcvFrom(msg_t *pM) { if(pM == NULL) - return ""; + return UCHAR_CONSTANT(""); else if(pM->pszRcvFrom == NULL) - return ""; + return UCHAR_CONSTANT(""); else - return (char*) pM->pszRcvFrom; + return pM->pszRcvFrom; } @@ -1488,13 +1466,13 @@ static int getAPPNAMELen(msg_t *pM) /* rgerhards 2008-09-10: set pszInputName in msg object */ -void MsgSetInputName(msg_t *pMsg, char* pszInputName) +void MsgSetInputName(msg_t *pMsg, uchar* pszInputName) { assert(pMsg != NULL); if(pMsg->pszInputName != NULL) free(pMsg->pszInputName); - pMsg->iLenInputName = strlen(pszInputName); + pMsg->iLenInputName = ustrlen(pszInputName); if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); } @@ -1502,13 +1480,12 @@ void MsgSetInputName(msg_t *pMsg, char* pszInputName) /* rgerhards 2004-11-16: set pszRcvFrom in msg object */ -void MsgSetRcvFrom(msg_t *pMsg, char* pszRcvFrom) +void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom) { assert(pMsg != NULL); - if(pMsg->pszRcvFrom != NULL) - free(pMsg->pszRcvFrom); + free(pMsg->pszRcvFrom); - pMsg->iLenRcvFrom = strlen(pszRcvFrom); + pMsg->iLenRcvFrom = ustrlen(pszRcvFrom); if((pMsg->pszRcvFrom = malloc(pMsg->iLenRcvFrom + 1)) != NULL) { memcpy(pMsg->pszRcvFrom, pszRcvFrom, pMsg->iLenRcvFrom + 1); } @@ -1559,13 +1536,12 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) * we need it. The rest of the code already knows how to handle an * unset HOSTNAME. */ -void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME) +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) { assert(pMsg != NULL); - if(pMsg->pszHOSTNAME != NULL) - free(pMsg->pszHOSTNAME); + free(pMsg->pszHOSTNAME); - pMsg->iLenHOSTNAME = strlen(pszHOSTNAME); + pMsg->iLenHOSTNAME = ustrlen(pszHOSTNAME); if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); else @@ -1790,7 +1766,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else if(!strcmp((char*) pName, "inputname")) { pRes = (char*) getInputName(pMsg); } else if(!strcmp((char*) pName, "fromhost")) { - pRes = getRcvFrom(pMsg); + pRes = (char*) getRcvFrom(pMsg); } else if(!strcmp((char*) pName, "fromhost-ip")) { pRes = (char*) getRcvFromIP(pMsg); } else if(!strcmp((char*) pName, "source") || !strcmp((char*) pName, "hostname")) { @@ -2517,13 +2493,13 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { - MsgSetRcvFrom(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFrom(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszHOSTNAME")) { - MsgSetHOSTNAME(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSStrucData")) { MsgSetStructuredData(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSAPPNAME")) { diff --git a/runtime/msg.h b/runtime/msg.h index c8350626..a14f6b15 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -158,7 +158,7 @@ char *getSeverity(msg_t *pM); char *getSeverityStr(msg_t *pM); char *getFacility(msg_t *pM); char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, char*); +void MsgSetInputName(msg_t *pMsg, uchar*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); @@ -171,15 +171,15 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); char *getTAG(msg_t *pM); int getHOSTNAMELen(msg_t *pM); char *getHOSTNAME(msg_t *pM); -char *getRcvFrom(msg_t *pM); +uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); char *getStructuredData(msg_t *pM); int getProgramNameLen(msg_t *pM); char *getProgramName(msg_t *pM); -void MsgSetRcvFrom(msg_t *pMsg, char* pszRcvFrom); +void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); -void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME); +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); diff --git a/runtime/netstrm.c b/runtime/netstrm.c index ffa1c578..459561bc 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -17,7 +17,7 @@ * Rainer Gerhards and Adiscon GmbH have agreed to permit using the code * under the terms of the GNU Lesser General Public License. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -114,6 +114,7 @@ AcceptConnReq(netstrm_t *pThis, netstrm_t **ppNew) ISOBJ_TYPE_assert(pThis, netstrm); assert(ppNew != NULL); +RUNLOG_STR("XXX: accept conn reqeust"); /* accept the new connection */ CHKiRet(pThis->Drvr.AcceptConnReq(pThis->pDrvrData, &pNewNsd)); /* construct our object so that we can use it... */ @@ -147,6 +148,7 @@ LstnInit(netstrms_t *pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t*), ISOBJ_TYPE_assert(pNS, netstrms); assert(fAddLstn != NULL); assert(pLstnPort != NULL); +RUNLOG_STR("XXX: Init Lstn"); CHKiRet(pNS->Drvr.LstnInit(pNS, pUsr, fAddLstn, pLstnPort, pLstnIP, iSessMax)); diff --git a/runtime/netstrm.h b/runtime/netstrm.h index 3ab790e8..b00dd223 100644 --- a/runtime/netstrm.h +++ b/runtime/netstrm.h @@ -31,6 +31,7 @@ struct netstrm_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ nsd_t *pDrvrData; /**< the driver's data elements (at most other places, this is called pNsd) */ nsd_if_t Drvr; /**< our stream driver */ + void *pUsr; /**< pointer to user-provided data structure */ netstrms_t *pNS; /**< pointer to our netstream subsystem object */ }; diff --git a/runtime/parser.c b/runtime/parser.c index b4ab0a3e..212d40f3 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -283,7 +283,7 @@ rsRetVal parseMsg(msg_t *pMsg) MsgSetUxTradMsg(pMsg, (char*) msg); if(pMsg->bParseHOSTNAME == 0) - MsgSetHOSTNAME(pMsg, (char*) pMsg->pszRcvFrom); + MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have * a traditional syslog message or one formatted according to syslog-protocol. diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index fd5a5371..8e8a9f2a 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -87,6 +87,8 @@ typedef struct tcpsrv_s tcpsrv_t; typedef struct vmstk_s vmstk_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ +typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? + /* some universal 64 bit define... */ typedef long long int64; typedef long long unsigned uint64; @@ -359,6 +361,7 @@ typedef enum rsObjectID rsObjID; /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); + #include "debug.h" #include "obj.h" diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h new file mode 100644 index 00000000..36d76a78 --- /dev/null +++ b/runtime/unicode-helper.h @@ -0,0 +1,54 @@ +/* This is the header file for unicode support. + * + * Currently, this is a dummy module. + * The following functions are wrappers which hopefully enable us to move + * from 8-bit chars to unicode with relative ease when we finally attack this + * + * Begun 2009-05-21 RGerhards + * + * Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_UNICODE_HELPER_H +#define INCLUDED_UNICODE_HELPER_H + +#include + +static inline int ustrcmp(uchar *psz1, uchar *psz2) +{ + return strcmp((char*) psz1, (char*) psz2); +} + +static inline int ustrlen(uchar *psz) +{ + return strlen((char*) psz); +} + +static inline uchar* ustrdup(uchar *psz) +{ + return (uchar*) strdup((char*)psz); +} + + +#define UCHAR_CONSTANT(x) ((uchar*) (x)) + +#endif /* multi-include protection */ +/* vim:set ai: + */ diff --git a/tcps_sess.c b/tcps_sess.c index ceb6142f..c564caea 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -185,6 +185,18 @@ SetTcpsrv(tcps_sess_t *pThis, tcpsrv_t *pSrv) } +/* set our parent listener info*/ +static rsRetVal +SetLstnInfo(tcps_sess_t *pThis, tcpLstnPortList_t *pLstnInfo) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcps_sess); + assert(pLstnInfo != NULL); + pThis->pLstnInfo = pLstnInfo; + RETiRet; +} + + static rsRetVal SetUsrP(tcps_sess_t *pThis, void *pUsr) { @@ -223,11 +235,11 @@ doSubmitMessage(tcps_sess_t *pThis) CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); pMsg->iLenRawMsg = pThis->iMsg; - MsgSetInputName(pMsg, (char*)pThis->pSrv->pszInputName); + MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; - MsgSetRcvFrom(pMsg, (char*)pThis->fromHost); + MsgSetRcvFrom(pMsg, pThis->fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); CHKiRet(submitMsg(pMsg)); @@ -457,6 +469,7 @@ CODESTARTobjQueryInterface(tcps_sess) pIf->SetUsrP = SetUsrP; pIf->SetTcpsrv = SetTcpsrv; + pIf->SetLstnInfo = SetLstnInfo; pIf->SetHost = SetHost; pIf->SetHostIP = SetHostIP; pIf->SetStrm = SetStrm; diff --git a/tcps_sess.h b/tcps_sess.h index 576466ff..2ef28264 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -31,7 +31,8 @@ struct tcpsrv_s; /* the tcps_sess object */ typedef struct tcps_sess_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - struct tcpsrv_s *pSrv; /* pointer back to my server (e.g. for callbacks) */ + tcpsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ + tcpLstnPortList_t *pLstnInfo; /* pointer back to listener info */ netstrm_t *pStrm; int iMsg; /* index of next char to store in msg */ int bAtStrtOfFram; /* are we at the very beginning of a new frame? */ @@ -60,13 +61,19 @@ BEGINinterface(tcps_sess) /* name must also be changed in ENDinterface macro! */ rsRetVal (*DataRcvd)(tcps_sess_t *pThis, char *pData, size_t iLen); /* set methods */ rsRetVal (*SetTcpsrv)(tcps_sess_t *pThis, struct tcpsrv_s *pSrv); + rsRetVal (*SetLstnInfo)(tcps_sess_t *pThis, tcpLstnPortList_t *pLstnInfo); rsRetVal (*SetUsrP)(tcps_sess_t*, void*); rsRetVal (*SetHost)(tcps_sess_t *pThis, uchar*); rsRetVal (*SetHostIP)(tcps_sess_t *pThis, uchar*); rsRetVal (*SetStrm)(tcps_sess_t *pThis, netstrm_t*); rsRetVal (*SetMsgIdx)(tcps_sess_t *pThis, int); ENDinterface(tcps_sess) -#define tcps_sessCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define tcps_sessCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* interface changes + * to version v2, rgerhards, 2009-05-22 + * - Data structures changed + * - SetLstnInfo entry point added + */ /* prototypes */ diff --git a/tcpsrv.c b/tcpsrv.c index 7af45d3c..bbd95058 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -66,6 +66,7 @@ #include "netstrm.h" #include "nssel.h" #include "errmsg.h" +#include "unicode-helper.h" MODULE_TYPE_LIB @@ -85,43 +86,61 @@ DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) -/* configure TCP listener settings. This is called during command - * line parsing. The argument following -t is supplied as an argument. - * The format of this argument is - * ", " - * Typically, there is no whitespace between port and session number. - * (but it may be...). - * NOTE: you can not use dbgprintf() in here - the dbgprintf() system is - * not yet initilized when this function is called. - * rgerhards, 2007-06-21 - * The port in cOptarg is handed over to us - the caller MUST NOT free it! +/* add new listener port to listener port list + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) +{ + tcpLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + +dbgprintf("XXX: tcpsrv.c add port %s, name '%s'\n", pszPort, pThis->pszInputName); + /* create entry */ + CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); + pEntry->pszPort = pszPort; + pEntry->pSrv = pThis; + CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + + /* and add to list */ + pEntry->pNext = pThis->pLstnPorts; + pThis->pLstnPorts = pEntry; + +finalize_it: + RETiRet; +} + + +/* configure TCP listener settings. + * Note: pszPort is handed over to us - the caller MUST NOT free it! * rgerhards, 2008-03-20 */ -static void -configureTCPListen(tcpsrv_t *pThis, char *cOptarg) +static rsRetVal +configureTCPListen(tcpsrv_t *pThis, uchar *pszPort) { - register int i; - register char *pArg = cOptarg; + int i; + uchar *pPort = pszPort; + DEFiRet; - assert(cOptarg != NULL); + assert(pszPort != NULL); ISOBJ_TYPE_assert(pThis, tcpsrv); /* extract port */ i = 0; - while(isdigit((int) *pArg)) { - i = i * 10 + *pArg++ - '0'; + while(isdigit((int) *pPort)) { + i = i * 10 + *pPort++ - '0'; } - if(pThis->TCPLstnPort != NULL) { - free(pThis->TCPLstnPort); - pThis->TCPLstnPort = NULL; - } - - if( i >= 0 && i <= 65535) { - pThis->TCPLstnPort = cOptarg; + if(i >= 0 && i <= 65535) { + CHKiRet(addNewLstnPort(pThis, pszPort)); } else { - errmsg.LogError(0, NO_ERRCODE, "Invalid TCP listen port %s - changed to 514.\n", cOptarg); + errmsg.LogError(0, NO_ERRCODE, "Invalid TCP listen port %s - ignored.\n", pszPort); } + +finalize_it: + RETiRet; } @@ -202,6 +221,8 @@ TCPSessGetNxtSess(tcpsrv_t *pThis, int iCurr) static void deinit_tcp_listener(tcpsrv_t *pThis) { int i; + tcpLstnPortList_t *pEntry; + tcpLstnPortList_t *pDel; ISOBJ_TYPE_assert(pThis, tcpsrv); @@ -219,7 +240,15 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) pThis->pSessions = NULL; /* just to make sure... */ } - free(pThis->TCPLstnPort); + /* free list of tcp listen ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + free(pEntry->pszPort); + free(pEntry->pszInputName); + pDel = pEntry; + pEntry = pEntry->pNext; + free(pDel); + } /* finally close our listen streams */ for(i = 0 ; i < pThis->iLstnMax ; ++i) { @@ -234,9 +263,11 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) static rsRetVal addTcpLstn(void *pUsr, netstrm_t *pLstn) { - tcpsrv_t *pThis = (tcpsrv_t*) pUsr; + tcpLstnPortList_t *pPortList = (tcpLstnPortList_t *) pUsr; + tcpsrv_t *pThis = pPortList->pSrv; DEFiRet; +dbgprintf("XXX: addTcpLst name %s\n", pPortList->pszInputName); ISOBJ_TYPE_assert(pThis, tcpsrv); ISOBJ_TYPE_assert(pLstn, netstrm); @@ -244,6 +275,7 @@ addTcpLstn(void *pUsr, netstrm_t *pLstn) ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); pThis->ppLstn[pThis->iLstnMax] = pLstn; + pThis->ppLstnPort[pThis->iLstnMax] = pPortList; ++pThis->iLstnMax; finalize_it: @@ -251,19 +283,20 @@ finalize_it: } -/* Initialize TCP sockets (for listener) and listens on them */ -static rsRetVal -create_tcp_socket(tcpsrv_t *pThis) +/* Initialize TCP listener socket for a single port + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +initTCPListener(tcpsrv_t *pThis, tcpLstnPortList_t *pPortEntry) { DEFiRet; uchar *TCPLstnPort; ISOBJ_TYPE_assert(pThis, tcpsrv); + assert(pPortEntry != NULL); - if(!strcmp((char*)pThis->TCPLstnPort, "0")) - TCPLstnPort = (uchar*)"514"; - // TODO: we need to enable the caller to set a port (based on who is - // using this, 514 may be totally unsuitable... --- rgerhards, 2008-04-22 + if(!ustrcmp(pPortEntry->pszPort, UCHAR_CONSTANT("0"))) + TCPLstnPort = UCHAR_CONSTANT("514"); /* use default - we can not do service db update, because there is * no IANA-assignment for syslog/tcp. In the long term, we might * re-use RFC 3195 port of 601, but that would probably break to @@ -271,10 +304,32 @@ create_tcp_socket(tcpsrv_t *pThis) * rgerhards, 2007-06-28 */ else - TCPLstnPort = (uchar*)pThis->TCPLstnPort; + TCPLstnPort = pPortEntry->pszPort; /* TODO: add capability to specify local listen address! */ - CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pThis, addTcpLstn, TCPLstnPort, NULL, pThis->iSessMax)); + CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pPortEntry, addTcpLstn, TCPLstnPort, NULL, pThis->iSessMax)); + +finalize_it: + RETiRet; +} + + +/* Initialize TCP sockets (for listener) and listens on them */ +static rsRetVal +create_tcp_socket(tcpsrv_t *pThis) +{ + tcpLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + + /* init all configured ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { +dbgprintf("XXX: tcpsrv.c create_tcp_socket do port %s\n", pEntry->pszPort); + CHKiRet(initTCPListener(pThis, pEntry)); + pEntry = pEntry->pNext; + } /* OK, we had success. Now it is also time to * initialize our connections @@ -296,7 +351,7 @@ finalize_it: /* Accept new TCP connection; make entry in session table. If there * is no more space left in the connection table, the new TCP * connection is immediately dropped. - * ppSess has a pointer to the newly created session, if it succeds. + * ppSess has a pointer to the newly created session, if it succeeds. * If it does not succeed, no session is created and ppSess is * undefined. If the user has provided an OnSessAccept Callback, * this one is executed immediately after creation of the @@ -304,7 +359,7 @@ finalize_it: * rgerhards, 2008-03-02 */ static rsRetVal -SessAccept(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm) +SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, netstrm_t *pStrm) { DEFiRet; tcps_sess_t *pSess = NULL; @@ -315,6 +370,7 @@ SessAccept(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm) uchar *fromHostIP = NULL; ISOBJ_TYPE_assert(pThis, tcpsrv); + assert(pLstnInfo != NULL); CHKiRet(netstrm.AcceptConnReq(pStrm, &pNewStrm)); @@ -328,6 +384,7 @@ SessAccept(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm) /* we found a free spot and can construct our session object */ CHKiRet(tcps_sess.Construct(&pSess)); CHKiRet(tcps_sess.SetTcpsrv(pSess, pThis)); + CHKiRet(tcps_sess.SetLstnInfo(pSess, pLstnInfo)); } /* OK, we have a "good" index... */ @@ -375,12 +432,10 @@ finalize_it: if(iRet != RS_RET_OK) { if(pSess != NULL) tcps_sess.Destruct(&pSess); - if(fromHostFQDN != NULL) - free(fromHostFQDN); - if(fromHostIP != NULL) - free(fromHostIP); if(pNewStrm != NULL) netstrm.Destruct(&pNewStrm); + free(fromHostFQDN); + free(fromHostIP); } RETiRet; @@ -444,7 +499,7 @@ Run(tcpsrv_t *pThis) CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); - SessAccept(pThis, &pNewSess, pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); --nfds; /* indicate we have processed one */ } } @@ -535,6 +590,7 @@ tcpsrvConstructFinalize(tcpsrv_t *pThis) /* set up listeners */ CHKmalloc(pThis->ppLstn = calloc(TCPLSTN_MAX_DEFAULT, sizeof(netstrm_t*))); + CHKmalloc(pThis->ppLstnPort = calloc(TCPLSTN_MAX_DEFAULT, sizeof(tcpLstnPortList_t*))); iRet = pThis->OpenLstnSocks(pThis); finalize_it: @@ -558,6 +614,7 @@ CODESTARTobjDestruct(tcpsrv) netstrms.Destruct(&pThis->pNS); free(pThis->pszDrvrAuthMode); free(pThis->ppLstn); + free(pThis->ppLstnPort); free(pThis->pszInputName); ENDobjDestruct(tcpsrv) @@ -674,11 +731,12 @@ SetInputName(tcpsrv_t *pThis, uchar *name) { uchar *pszName; DEFiRet; +dbgprintf("XXX: SetInputName: %s\n", name); ISOBJ_TYPE_assert(pThis, tcpsrv); if(name == NULL) pszName = NULL; else - CHKmalloc(pszName = (uchar*)strdup((char*)name)); + CHKmalloc(pszName = ustrdup(name)); free(pThis->pszInputName); pThis->pszInputName = pszName; finalize_it: @@ -708,7 +766,7 @@ SetDrvrAuthMode(tcpsrv_t *pThis, uchar *mode) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcpsrv); - CHKmalloc(pThis->pszDrvrAuthMode = (uchar*)strdup((char*)mode)); + CHKmalloc(pThis->pszDrvrAuthMode = ustrdup(mode)); finalize_it: RETiRet; } @@ -763,7 +821,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->ConstructFinalize = tcpsrvConstructFinalize; pIf->Destruct = tcpsrvDestruct; - pIf->SessAccept = SessAccept; + //pIf->SessAccept = SessAccept; pIf->configureTCPListen = configureTCPListen; pIf->create_tcp_socket = create_tcp_socket; pIf->Run = Run; diff --git a/tcpsrv.h b/tcpsrv.h index e5ecb865..0f7dd6c6 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -32,6 +32,15 @@ typedef enum ETCPsyslogFramingAnomaly { frame_CiscoIOS = 2 } eTCPsyslogFramingAnomaly; + +/* list of tcp listen ports */ +struct tcpLstnPortList_s { + uchar *pszPort; /**< the ports the listener shall listen on */ + uchar *pszInputName; /**< value to be used as input name */ + tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ + tcpLstnPortList_t *pNext; /**< next port or NULL */ +}; + #define TCPSRV_NO_ADDTL_DELIMITER -1 /* specifies that no additional delimiter is to be used in TCP framing */ /* the tcpsrv object */ @@ -44,8 +53,9 @@ struct tcpsrv_s { permittedPeers_t *pPermPeers;/**< driver's permitted peers */ int iLstnMax; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ + tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ int iSessMax; /**< max number of sessions supported */ - char *TCPLstnPort; /**< the port the listener shall listen on */ + tcpLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ int addtlFrameDelim; /**< additional frame delimiter for plain TCP syslog framing (e.g. to handle NetScreen) */ tcps_sess_t **pSessions;/**< array of all of our sessions */ void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ @@ -70,8 +80,8 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Construct)(tcpsrv_t **ppThis); rsRetVal (*ConstructFinalize)(tcpsrv_t __attribute__((unused)) *pThis); rsRetVal (*Destruct)(tcpsrv_t **ppThis); - void (*configureTCPListen)(tcpsrv_t*, char *cOptarg); - rsRetVal (*SessAccept)(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm); + rsRetVal (*configureTCPListen)(tcpsrv_t*, uchar *pszPort); + //rsRetVal (*SessAccept)(tcpsrv_t *pThis, tcpLstnPortList_t*, tcps_sess_t **ppSess, netstrm_t *pStrm); rsRetVal (*create_tcp_socket)(tcpsrv_t *pThis); rsRetVal (*Run)(tcpsrv_t *pThis); /* set methods */ diff --git a/tests/Makefile.am b/tests/Makefile.am index ed415a1e..bc1849f0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,8 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh if ENABLE_OMSTDOUT -TESTS += omod-if-array.sh parsertest.sh +TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid @@ -29,11 +29,22 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rfc5424-4.parse1 \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ + killrsyslog.sh \ parsertest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ + imtcp-multiport.sh \ + testsuites/imtcp-multiport.conf \ manytcp.sh \ testsuites/manytcp.conf \ + fieldtest.sh \ + testsuites/field.conf \ + testsuites/1.field1 \ + inputname.sh \ + testsuites/inputname_imtcp.conf \ + testsuites/1.inputname_imtcp_12514 \ + testsuites/1.inputname_imtcp_12515 \ + testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ cfg.sh diff --git a/tests/fieldtest.sh b/tests/fieldtest.sh new file mode 100755 index 00000000..7a646f00 --- /dev/null +++ b/tests/fieldtest.sh @@ -0,0 +1,13 @@ +echo test fieldtest via udp +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -tfield1 -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test fieldtest via tcp +./nettester -tfield1 -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh new file mode 100755 index 00000000..aa1f15e8 --- /dev/null +++ b/tests/imtcp-multiport.sh @@ -0,0 +1,80 @@ +# Test for multiple ports in imtcp +# This test checks if multiple tcp listener ports are correctly +# handled by imtcp +# +# NOTE: this test must (and can) be enhanced when we merge in the +# upgraded tcpflood program +# +# added 2009-05-22 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo testing imtcp multiple listeners +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13514 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 2 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +# +# +# ### now complete new cycle with other port ### +# +# +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13515 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 2 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +# +# +# ### now complete new cycle with other port ### +# +# +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13516 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 2 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log diff --git a/tests/inputname.sh b/tests/inputname.sh new file mode 100755 index 00000000..7d9ea111 --- /dev/null +++ b/tests/inputname.sh @@ -0,0 +1,20 @@ +echo testing $InputTCPServerInputName directive +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +echo port 12514 +./nettester -tinputname_imtcp_12514 -cinputname_imtcp -itcp -p12514 +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo port 12515 +./nettester -tinputname_imtcp_12515 -cinputname_imtcp -itcp -p12515 +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo port 12516 +./nettester -tinputname_imtcp_12516 -cinputname_imtcp -itcp -p12516 +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/killrsyslog.sh b/tests/killrsyslog.sh new file mode 100755 index 00000000..b1be757b --- /dev/null +++ b/tests/killrsyslog.sh @@ -0,0 +1,7 @@ +#check if rsyslog instance exists and, if so, kill it +if [ -e "rsyslog.pid" ] +then + echo rsyslog.pid exists, trying to shut down rsyslogd process `cat rsyslog.pid`. + kill `cat rsyslog.pid` + sleep 1 +fi diff --git a/tests/nettester.c b/tests/nettester.c index 37183ac9..566f553b 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -45,16 +45,36 @@ #include #include #include +#include #define EXIT_FAILURE 1 #define INVALID_SOCKET -1 /* Name of input file, must match $IncludeConfig in test suite .conf files */ #define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ -static enum { inputUDP, inputTCP } inputMode; /* input for which tests are to be run */ +typedef enum { inputUDP, inputTCP } inputMode_t; +inputMode_t inputMode = inputTCP; /* input for which tests are to be run */ static pid_t rsyslogdPid = 0; /* pid of rsyslog instance being tested */ static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ -static char *testSuite; /* name of current test suite */ +static char *testSuite = NULL; /* name of current test suite */ +static int iPort = 12514; /* port which shall be used for sending data */ +static char* pszCustomConf = NULL; /* custom config file, use -c conf to specify */ +static int verbose = 0; /* verbose output? -v option */ + + +/* provide user-friednly name of input mode + */ +static char *inputMode2Str(inputMode_t mode) +{ + char *pszMode; + + if(mode == inputUDP) + pszMode = "udp"; + else + pszMode = "tcp"; + + return pszMode; +} void readLine(int fd, char *ln) @@ -94,7 +114,7 @@ tcpSend(char *buf, int lenBuf) memset((char *) &addr, 0, sizeof(addr)); addr.sin_family = AF_INET; - addr.sin_port = htons(13514); + addr.sin_port = htons(iPort); if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { fprintf(stderr, "inet_aton() failed\n"); return(1); @@ -139,7 +159,7 @@ udpSend(char *buf, int lenBuf) memset((char *) &si_other, 0, sizeof(si_other)); si_other.sin_family = AF_INET; - si_other.sin_port = htons(12514); + si_other.sin_port = htons(iPort); if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { fprintf(stderr, "inet_aton() failed\n"); return(1); @@ -169,9 +189,14 @@ int openPipe(char *configFile, pid_t *pid, int *pfd) "-M../runtime/.libs:../.libs", NULL }; char confFile[1024]; char *newenviron[] = { NULL }; + /* debug aide... + char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout", + "RSYSLOG_DEBUGLOG=tmp", NULL }; + */ - sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, + (pszCustomConf == NULL) ? configFile : pszCustomConf); newargv[1] = confFile; if (pipe(pipefd) == -1) { @@ -291,12 +316,12 @@ doTests(int fd, char *files) ++iTests; /* all regular files are run through the test logic. Symlinks don't work. */ if(S_ISREG(fileInfo.st_mode)) { /* config file */ - printf("processing test case '%s' ... ", testFile); + if(verbose) printf("processing test case '%s' ... ", testFile); ret = processTestFile(fd, testFile); if(ret == 0) { - printf("successfully completed\n"); + if(verbose) printf("successfully completed\n"); } else { - printf("failed!\n"); + if(verbose) printf("failed!\n"); ++iFailed; } } @@ -334,36 +359,54 @@ void doAtExit(void) int main(int argc, char *argv[]) { int fd; + int opt; int ret = 0; FILE *fp; char buf[4096]; char testcases[4096]; - if(argc != 3) { - printf("Invalid call of nettester\n"); - printf("Usage: nettester testsuite-name input\n"); - printf(" input = udp|tcp\n"); - exit(1); + while((opt = getopt(argc, argv, "c:i:p:t:v")) != EOF) { + switch((char)opt) { + case 'c': + pszCustomConf = optarg; + break; + case 'i': + if(!strcmp(optarg, "udp")) + inputMode = inputUDP; + else if(!strcmp(optarg, "tcp")) + inputMode = inputTCP; + else { + printf("error: unsupported input mode '%s'\n", optarg); + exit(1); + } + break; + case 'p': + iPort = atoi(optarg); + break; + case 't': + testSuite = optarg; + break; + case 'v': + verbose = 1; + break; + default:printf("Invalid call of nettester, invalid option '%c'.\n", opt); + printf("Usage: nettester -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n"); + exit(1); + } } - atexit(doAtExit); - - testSuite = argv[1]; - - if(!strcmp(argv[2], "udp")) - inputMode = inputUDP; - else if(!strcmp(argv[2], "tcp")) - inputMode = inputTCP; - else { - printf("error: unsupported input mode '%s'\n", argv[2]); + if(testSuite == NULL) { + printf("error: no testsuite given, need to specify -t testsuite!\n"); exit(1); } + atexit(doAtExit); + if((srcdir = getenv("srcdir")) == NULL) srcdir = "."; - printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s)\n", - srcdir, testSuite, argv[2]); + if(verbose) printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s/%d)\n", + srcdir, testSuite, inputMode2Str(inputMode), iPort); /* create input config file */ if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { @@ -373,15 +416,15 @@ int main(int argc, char *argv[]) } if(inputMode == inputUDP) { fputs("$ModLoad ../plugins/imudp/.libs/imudp\n", fp); - fputs("$UDPServerRun 12514\n", fp); + fprintf(fp, "$UDPServerRun %d\n", iPort); } else { fputs("$ModLoad ../plugins/imtcp/.libs/imtcp\n", fp); - fputs("$InputTCPServerRun 13514\n", fp); + fprintf(fp, "$InputTCPServerRun %d\n", iPort); } fclose(fp); /* start to be tested rsyslogd */ - openPipe(argv[1], &rsyslogdPid, &fd); + openPipe(testSuite, &rsyslogdPid, &fd); readLine(fd, buf); /* generate filename */ @@ -389,6 +432,6 @@ int main(int argc, char *argv[]) if(doTests(fd, testcases) != 0) ret = 1; - printf("End of nettester run (%d).\n", ret); + if(verbose) printf("End of nettester run (%d).\n", ret); exit(ret); } diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index fd845b4d..7b4b5611 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,11 +1,13 @@ echo test omod-if-array via udp -./nettester omod-if-array udp +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -tomod-if-array -iudp -p4711 if [ "$?" -ne "0" ]; then exit 1 fi echo test omod-if-array via tcp -./nettester omod-if-array tcp +./nettester -tomod-if-array -itcp if [ "$?" -ne "0" ]; then exit 1 fi diff --git a/tests/parsertest.sh b/tests/parsertest.sh index a6b7d45c..152d8b60 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,11 +1,13 @@ echo test parsertest via udp -./nettester parse1 udp +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -tparse1 -iudp if [ "$?" -ne "0" ]; then exit 1 fi echo test parsertest via tcp -./nettester parse1 tcp +./nettester -tparse1 -itcp if [ "$?" -ne "0" ]; then exit 1 fi diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 8dbc201b..c3c9c871 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -154,8 +154,6 @@ int sendMessages(void) int lenBuf; int lenSend; char buf[2048]; - char msgBuf[128]; - size_t lenMsg; srand(time(NULL)); /* seed is good enough for our needs */ diff --git a/tests/testsuites/1.field1 b/tests/testsuites/1.field1 new file mode 100644 index 00000000..54751171 --- /dev/null +++ b/tests/testsuites/1.field1 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: DROP_url_www.sina.com.cn:IN=eth1 OUT=eth0 SRC=192.168.10.78 DST=61.172.201.194 LEN=1182 TOS=0x00 PREC=0x00 TTL=63 ID=14368 DF PROTO=TCP SPT=33343 DPT=80 WINDOW=92 RES=0x00 ACK PSH URGP=0 +DROP_url_www.sina.com.cn:IN=eth1 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.inputname_imtcp_12514 b/tests/testsuites/1.inputname_imtcp_12514 new file mode 100644 index 00000000..178b1724 --- /dev/null +++ b/tests/testsuites/1.inputname_imtcp_12514 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: MSG +12514 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.inputname_imtcp_12515 b/tests/testsuites/1.inputname_imtcp_12515 new file mode 100644 index 00000000..d616098b --- /dev/null +++ b/tests/testsuites/1.inputname_imtcp_12515 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: MSG +12515 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.inputname_imtcp_12516 b/tests/testsuites/1.inputname_imtcp_12516 new file mode 100644 index 00000000..8e6997ce --- /dev/null +++ b/tests/testsuites/1.inputname_imtcp_12516 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: MSG +12516 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/field1.conf b/tests/testsuites/field1.conf new file mode 100644 index 00000000..1ff833dd --- /dev/null +++ b/tests/testsuites/field1.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%msg:F,32:2%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf new file mode 100644 index 00000000..00b63cb2 --- /dev/null +++ b/tests/testsuites/imtcp-multiport.conf @@ -0,0 +1,13 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-05-22 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 +$InputTCPServerRun 13515 +$InputTCPServerRun 13516 + +$ErrorMessagesToStderr off + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/inputname_imtcp.conf b/tests/testsuites/inputname_imtcp.conf new file mode 100644 index 00000000..a25eab37 --- /dev/null +++ b/tests/testsuites/inputname_imtcp.conf @@ -0,0 +1,19 @@ +# This is a special case, thus we define the inputs ourselfs +$ModLoad ../plugins/omstdout/.libs/omstdout + +$ModLoad ../plugins/imtcp/.libs/imtcp + +$InputTCPServerInputname 12514 +$InputTCPServerRun 12514 + +$InputTCPServerInputname 12515 +$InputTCPServerRun 12515 + +$InputTCPServerInputname 12516 +$InputTCPServerRun 12516 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%inputname%\n" +*.* :omstdout:;fmt diff --git a/tools/syslogd.c b/tools/syslogd.c index 3a751a30..dc5b8fee 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -137,6 +137,7 @@ #include "datetime.h" #include "parser.h" #include "sysvar.h" +#include "unicode-helper.h" /* definitions for objects we access */ DEFobjCurrIf(obj) @@ -620,7 +621,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) - MsgSetInputName(pMsg, (char*) pszInputName); + MsgSetInputName(pMsg, pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsg(pMsg, (char*)msg); @@ -649,8 +650,8 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f * being the local host). rgerhards 2004-11-16 */ if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) - MsgSetHOSTNAME(pMsg, (char*)hname); - MsgSetRcvFrom(pMsg, (char*)hname); + MsgSetHOSTNAME(pMsg, hname); + MsgSetRcvFrom(pMsg, hname); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); /* rgerhards 2004-11-19: well, well... we've now seen that we @@ -928,12 +929,12 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputName(pMsg, "rsyslogd"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd")); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); - MsgSetHOSTNAME(pMsg, (char*)glbl.GetLocalHostName()); - MsgSetRcvFrom(pMsg, (char*)glbl.GetLocalHostName()); - MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); + MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* check if we have an error code associated and, if so, * adjust the tag. -- r5gerhards, 2008-06-27 */ @@ -1233,9 +1234,9 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) * SP-terminated or any other error occurs. * rger, 2005-11-24 */ -static int parseRFCField(char **pp2parse, char *pResult) +static int parseRFCField(uchar **pp2parse, uchar *pResult) { - char *p2parse; + uchar *p2parse; int iRet = 0; assert(pp2parse != NULL); @@ -1271,9 +1272,9 @@ static int parseRFCField(char **pp2parse, char *pResult) * SP-terminated or any other error occurs. * rger, 2005-11-24 */ -static int parseRFCStructuredData(char **pp2parse, char *pResult) +static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult) { - char *p2parse; + uchar *p2parse; int bCont = 1; int iRet = 0; @@ -1340,14 +1341,14 @@ static int parseRFCStructuredData(char **pp2parse, char *pResult) */ int parseRFCSyslogMsg(msg_t *pMsg, int flags) { - char *p2parse; - char *pBuf; + uchar *p2parse; + uchar *pBuf; int bContParse = 1; BEGINfunc assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); - p2parse = (char*) pMsg->pszUxTradMsg; + p2parse = pMsg->pszUxTradMsg; /* do a sanity check on the version and eat it */ assert(p2parse[0] == '1' && p2parse[1] == ' '); @@ -1358,7 +1359,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) * message, so we can not run into any troubles. I think this is * more wise then to use individual buffers. */ - if((pBuf = malloc(sizeof(char)* strlen(p2parse) + 1)) == NULL) + if((pBuf = malloc(sizeof(uchar) * ustrlen(p2parse) + 1)) == NULL) return 1; /* IMPORTANT NOTE: @@ -1393,29 +1394,29 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) /* APP-NAME */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetAPPNAME(pMsg, pBuf); + MsgSetAPPNAME(pMsg, (char*)pBuf); } /* PROCID */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetPROCID(pMsg, pBuf); + MsgSetPROCID(pMsg, (char*)pBuf); } /* MSGID */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetMSGID(pMsg, pBuf); + MsgSetMSGID(pMsg, (char*)pBuf); } /* STRUCTURED-DATA */ if(bContParse) { parseRFCStructuredData(&p2parse, pBuf); - MsgSetStructuredData(pMsg, pBuf); + MsgSetStructuredData(pMsg, (char*)pBuf); } /* MSG */ - MsgSetMSG(pMsg, p2parse); + MsgSetMSG(pMsg, (char*)p2parse); free(pBuf); ENDfunc @@ -1438,7 +1439,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) */ int parseLegacySyslogMsg(msg_t *pMsg, int flags) { - char *p2parse; + uchar *p2parse; char *pBuf; char *pWork; cstr_t *pStrB; @@ -1448,7 +1449,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); - p2parse = (char*) pMsg->pszUxTradMsg; + p2parse = pMsg->pszUxTradMsg; /* Check to see if msg contains a timestamp. We start by assuming * that the message timestamp is the time of reciption (which we @@ -1503,7 +1504,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) /* the memory allocated is far too much in most cases. But on the plus side, * it is quite fast... - rgerhards, 2007-09-20 */ - if((pBuf = malloc(sizeof(char)* (strlen(p2parse) +1))) == NULL) + if((pBuf = malloc(sizeof(char)* (ustrlen(p2parse) +1))) == NULL) return 1; pWork = pBuf; /* this is the actual parsing loop */ @@ -1609,7 +1610,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } /* The rest is the actual MSG */ - MsgSetMSG(pMsg, p2parse); + MsgSetMSG(pMsg, (char*)p2parse); ENDfunc return 0; /* all ok */ @@ -1919,10 +1920,10 @@ static void doDie(int sig) # define MSG1 "DoDie called.\n" # define MSG2 "DoDie called 5 times - unconditional exit\n" static int iRetries = 0; /* debug aid */ - if(Debug || NoFork) + if(Debug) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { - if(Debug || NoFork) + if(Debug) write(1, MSG2, sizeof(MSG2) - 1); abort(); } -- cgit From 7adb9877f0c08f929d89f436103dfade03e8ea07 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 May 2009 17:40:24 +0200 Subject: fixing some nits with the build system --- runtime/Makefile.am | 2 +- tests/Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 8331882f..bc03c4a7 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -7,7 +7,7 @@ pkglib_LTLIBRARIES = librsyslog_la_SOURCES = \ rsyslog.c \ rsyslog.h \ - unicode_helper.h \ + unicode-helper.h \ atomic.h \ syslogd-types.h \ module-template.h \ diff --git a/tests/Makefile.am b/tests/Makefile.am index bc1849f0..c0e629d3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -38,7 +38,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ manytcp.sh \ testsuites/manytcp.conf \ fieldtest.sh \ - testsuites/field.conf \ + testsuites/field1.conf \ testsuites/1.field1 \ inputname.sh \ testsuites/inputname_imtcp.conf \ -- cgit From eb1615068c6a704287eda732d287280df4cc4c44 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 10:47:22 +0200 Subject: added new testing module imdiag which enables to talk to the rsyslog core at runtime. The current implementation is only a beginning, but can be expanded over time --- ChangeLog | 3 + Makefile.am | 2 +- dirty.h | 1 + plugins/imdiag/imdiag.c | 247 +++++++++++++++++++++++++++++++--------- runtime/netstrm.c | 1 - runtime/rsyslog.h | 1 + tcps_sess.c | 27 ++++- tcps_sess.h | 8 +- tcpsrv.c | 26 +++-- tcpsrv.h | 5 +- tests/Makefile.am | 1 + tests/diskqueue.sh | 2 +- tests/fieldtest.sh | 2 +- tests/inputname.sh | 2 +- tests/manytcp.sh | 2 +- tests/omod-if-array.sh | 2 +- tests/parsertest.sh | 2 +- tests/testsuites/diskqueue.conf | 3 + tests/testsuites/manytcp.conf | 3 + tests/waitqueueempty.sh | 4 + tools/syslogd.c | 27 ++++- 21 files changed, 289 insertions(+), 82 deletions(-) create mode 100755 tests/waitqueueempty.sh diff --git a/ChangeLog b/ChangeLog index b7a0a67a..a0357889 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,9 @@ Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs - build system improvements - thanks to Michael Biebl +- added new testing module imdiag, which enables to talk to the + rsyslog core at runtime. The current implementation is only a + beginning, but can be expanded over time --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/Makefile.am b/Makefile.am index a5cf879c..8d57700f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,5 +131,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag ACLOCAL_AMFLAGS = -I m4 diff --git a/dirty.h b/dirty.h index db9bc31b..6d585753 100644 --- a/dirty.h +++ b/dirty.h @@ -32,6 +32,7 @@ rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); int parseRFCSyslogMsg(msg_t *pMsg, int flags); int parseLegacySyslogMsg(msg_t *pMsg, int flags); +rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 90c5d9ee..40f94692 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -26,8 +26,8 @@ * * A copy of the GPL can be found in the file "COPYING" in this distribution. */ - #include "config.h" +#include #include #include #include @@ -43,114 +43,247 @@ #include #endif #include "rsyslog.h" -//#include "dirty.h" +#include "dirty.h" #include "cfsysline.h" #include "module-template.h" +#include "unicode-helper.h" #include "net.h" #include "netstrm.h" -#include "netstrms.h" #include "errmsg.h" +#include "tcpsrv.h" +#include "srUtils.h" +#include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT /* static data */ DEF_IMOD_STATIC_DATA +DEFobjCurrIf(tcpsrv) +DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) -DEFobjCurrIf(netstrms) DEFobjCurrIf(errmsg) /* Module static data */ -netstrms_t *pNS; /**< pointer to network stream subsystem */ -netstrm_t *arrLstn[10]; /**< our netstream listners */ -int iLstnMax = 0; /**< max nbr of listeners currently supported */ +static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ +static permittedPeers_t *pPermPeersRoot = NULL; /* config settings */ +static int iTCPSessMax = 20; /* max number of sessions */ +static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ +static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ +static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ + + +/* callbacks */ +/* this shall go into a specific ACL module! */ +static int +isPermittedHost(struct sockaddr __attribute__((unused)) *addr, char __attribute__((unused)) *fromHostFQDN, + void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) +{ + return 1; /* TODO: implement ACLs ... or via some other way? */ +} -/* add a listen socket to our listen socket array. This is a callback - * invoked from the netstrm class. -- rgerhards, 2008-04-23 +static rsRetVal +doOpenLstnSocks(tcpsrv_t *pSrv) +{ + ISOBJ_TYPE_assert(pSrv, tcpsrv); + return tcpsrv.create_tcp_socket(pSrv); +} + + +static rsRetVal +doRcvData(tcps_sess_t *pSess, char *buf, size_t lenBuf, ssize_t *piLenRcvd) +{ + DEFiRet; + assert(pSess != NULL); + assert(piLenRcvd != NULL); + + *piLenRcvd = lenBuf; + CHKiRet(netstrm.Rcv(pSess->pStrm, (uchar*) buf, piLenRcvd)); +finalize_it: + RETiRet; +} + +static rsRetVal +onRegularClose(tcps_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + /* process any incomplete frames left over */ + tcps_sess.PrepareClose(pSess); + /* Session closed */ + tcps_sess.Close(pSess); + RETiRet; +} + + +static rsRetVal +onErrClose(tcps_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + tcps_sess.Close(pSess); + RETiRet; +} + +/* ------------------------------ end callbacks ------------------------------ */ + + +/* This function waits until the main queue is drained (size = 0) */ static rsRetVal -addTcpLstn(netstrm_t *pLstn) +waitMainQEmpty(void) { + int iMsgQueueSize; DEFiRet; - ISOBJ_TYPE_assert(pLstn, netstrm); + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + while(iMsgQueueSize > 0) { + srSleep(0,2); /* wait a little bit */ + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + } + +finalize_it: + RETiRet; +} + - if((unsigned)iLstnMax >= sizeof(arrLstn)/sizeof(netstrm_t*)) - ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); +/* Function to handle received messages. This is our core function! + * rgerhards, 2009-05-24 + */ +static rsRetVal +OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) +{ + ssize_t len; + int iMsgQueueSize; + uchar *pszMsg; + uchar buf[1024]; + DEFiRet; - arrLstn[iLstnMax] = pLstn; - ++iLstnMax; + assert(pSess != NULL); + assert(pRcv != NULL); + + /* NOTE: pRcv is NOT a C-String but rather an array of characters + * WITHOUT a termination \0 char. So we need to convert it to one + * before proceeding. + */ + CHKmalloc(pszMsg = malloc(sizeof(uchar) * (iLenMsg + 1))); + memcpy(pszMsg, pRcv, iLenMsg); + pszMsg[iLenMsg] = '\0'; + + if(!ustrcmp(pszMsg, UCHAR_CONSTANT("GetMainMsgQueueSize"))) { + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "%d\n", iMsgQueueSize); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + } else if(!ustrcmp(pszMsg, UCHAR_CONSTANT("WaitMainQueueEmpty"))) { + CHKiRet(waitMainQEmpty()); + len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "mainqueue empty\n"); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + } else { + len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "unkown command '%s'\n", pszMsg); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + } finalize_it: RETiRet; } -/* initialize network stream subsystem */ +/* set permitted peer -- rgerhards, 2008-05-19 + */ static rsRetVal -initNetstrm(void) +setPermittedPeer(void __attribute__((unused)) *pVal, uchar *pszID) +{ + DEFiRet; + CHKiRet(net.AddPermittedPeer(&pPermPeersRoot, pszID)); + free(pszID); /* no longer needed, but we need to free as of interface def */ +finalize_it: + RETiRet; +} + + +static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; - /* prepare network stream subsystem */ - CHKiRet(netstrms.Construct(&pNS)); - CHKiRet(netstrms.SetDrvrMode(pNS, 0)); /* always plain text */ - //CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); - //CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); - // TODO: set driver! - CHKiRet(netstrms.ConstructFinalize(&pNS)); + if(pOurTcpsrv == NULL) { + CHKiRet(tcpsrv.Construct(&pOurTcpsrv)); + CHKiRet(tcpsrv.SetSessMax(pOurTcpsrv, iTCPSessMax)); + CHKiRet(tcpsrv.SetCBIsPermittedHost(pOurTcpsrv, isPermittedHost)); + CHKiRet(tcpsrv.SetCBRcvData(pOurTcpsrv, doRcvData)); + CHKiRet(tcpsrv.SetCBOpenLstnSocks(pOurTcpsrv, doOpenLstnSocks)); + CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); + CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); + CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); + CHKiRet(tcpsrv.SetOnMsgReceive(pOurTcpsrv, OnMsgReceived)); + /* now set optional params, but only if they were actually configured */ + if(pszStrmDrvrAuthMode != NULL) { + CHKiRet(tcpsrv.SetDrvrAuthMode(pOurTcpsrv, pszStrmDrvrAuthMode)); + } + if(pPermPeersRoot != NULL) { + CHKiRet(tcpsrv.SetDrvrPermPeers(pOurTcpsrv, pPermPeersRoot)); + } + } - /* set up listeners */ - CHKiRet(netstrm.LstnInit(pNS, NULL, addTcpLstn, "127.0.0.1", (uchar*)"44514", 1)); + /* initialized, now add socket */ + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? + UCHAR_CONSTANT("imdiag") : pszInputName)); + tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); finalize_it: if(iRet != RS_RET_OK) { - if(pNS != NULL) - netstrms.Destruct(&pNS); + errmsg.LogError(0, NO_ERRCODE, "error %d trying to add listener", iRet); + if(pOurTcpsrv != NULL) + tcpsrv.Destruct(&pOurTcpsrv); } RETiRet; } - -/* This function is called to gather input. In our case, it is a bit abused - * to drive the listener loop for the diagnostics code. +/* This function is called to gather input. */ BEGINrunInput CODESTARTrunInput + CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); + iRet = tcpsrv.Run(pOurTcpsrv); +finalize_it: ENDrunInput /* initialize and return if will run or not */ BEGINwillRun CODESTARTwillRun - iRet = initNetstrm(); + /* first apply some config settings */ + if(pOurTcpsrv == NULL) + ABORT_FINALIZE(RS_RET_NO_RUN); +finalize_it: ENDwillRun BEGINafterRun - int i; CODESTARTafterRun /* do cleanup here */ - /* finally close our listen streams */ - for(i = 0 ; i < iLstnMax ; ++i) { - netstrm.Destruct(arrLstn[i]); - } - - /* destruct netstream subsystem */ - netstrms.Destruct(pNS); ENDafterRun BEGINmodExit CODESTARTmodExit + if(pOurTcpsrv != NULL) + iRet = tcpsrv.Destruct(&pOurTcpsrv); + + if(pPermPeersRoot != NULL) { + net.DestructPermittedPeers(&pPermPeersRoot); + } + /* release objects we used */ objRelease(net, LM_NET_FILENAME); - objRelease(netstrm, DONT_LOAD_LIB); - objRelease(netstrms, LM_NETSTRMS_FILENAME); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(tcps_sess, LM_TCPSRV_FILENAME); + objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -158,6 +291,14 @@ ENDmodExit static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { + iTCPSessMax = 200; + iStrmDrvrMode = 0; + free(pszInputName); + pszInputName = NULL; + if(pszStrmDrvrAuthMode != NULL) { + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; + } return RS_RET_OK; } @@ -173,26 +314,28 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + pOurTcpsrv = NULL; /* request objects we use */ CHKiRet(objUse(net, LM_NET_FILENAME)); - CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); - CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); + CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); + CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* register config file handlers */ -#if 0 - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverrun", 0, eCmdHdlrGetWord, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverrun"), 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpmaxsessions", 0, eCmdHdlrInt, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagmaxsessions"), 0, eCmdHdlrInt, NULL, &iTCPSessMax, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdrivermode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverstreamdrivermode"), 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverauthmode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverstreamdriverauthmode"), 0, eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverstreamdriverpermittedpeer"), 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); -#endif - CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverinputname"), 0, + eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/runtime/netstrm.c b/runtime/netstrm.c index 459561bc..f6a8de7f 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -148,7 +148,6 @@ LstnInit(netstrms_t *pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t*), ISOBJ_TYPE_assert(pNS, netstrms); assert(fAddLstn != NULL); assert(pLstnPort != NULL); -RUNLOG_STR("XXX: Init Lstn"); CHKiRet(pNS->Drvr.LstnInit(pNS, pUsr, fAddLstn, pLstnPort, pLstnIP, iSessMax)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 8e8a9f2a..77d845fd 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -84,6 +84,7 @@ typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; +typedef struct tcps_sess_s tcps_sess_t; typedef struct vmstk_s vmstk_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ diff --git a/tcps_sess.c b/tcps_sess.c index c564caea..c4548804 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -58,6 +58,7 @@ static int iMaxLine; /* maximum size of a single message */ /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); +static rsRetVal defaultDoSubmitMessage(tcps_sess_t *pThis, uchar*, int); /* Standard-Constructor */ @@ -65,6 +66,7 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->iMsg = 0; /* just make sure... */ pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ + pThis->DoSubmitMessage = defaultDoSubmitMessage; /* now allocate the message reception buffer */ CHKmalloc(pThis->pMsg = (uchar*) malloc(sizeof(uchar) * iMaxLine + 1)); finalize_it: @@ -206,6 +208,15 @@ SetUsrP(tcps_sess_t *pThis, void *pUsr) } +static rsRetVal +SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)) +{ + DEFiRet; + pThis->DoSubmitMessage = OnMsgReceive; + RETiRet; +} + + /* This is a helper for submitting the message to the rsyslog core. * It does some common processing, including resetting the various * state variables to a "processed" state. @@ -217,8 +228,11 @@ SetUsrP(tcps_sess_t *pThis, void *pUsr) * rgerhards, 2009-04-23 */ static rsRetVal -doSubmitMessage(tcps_sess_t *pThis) +defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) { +// TODO: make calling this overridable so that the diag module can ask to be called +// and so it can do its work right in this entry point (but we need to check that +// we have the capability to send a reply at this point). msg_t *pMsg; struct syslogTime stTime; time_t ttGenTime; @@ -233,7 +247,7 @@ doSubmitMessage(tcps_sess_t *pThis) CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); - memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); + memcpy(pMsg->pszRawMsg, pszMsg, iLenMsg); pMsg->iLenRawMsg = pThis->iMsg; MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); @@ -291,7 +305,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); } finalize_it: @@ -372,7 +386,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -383,7 +397,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -400,7 +414,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); pThis->inputState = eAtStrtFram; } } @@ -474,6 +488,7 @@ CODESTARTobjQueryInterface(tcps_sess) pIf->SetHostIP = SetHostIP; pIf->SetStrm = SetStrm; pIf->SetMsgIdx = SetMsgIdx; + pIf->SetOnMsgReceive = SetOnMsgReceive; finalize_it: ENDobjQueryInterface(tcps_sess) diff --git a/tcps_sess.h b/tcps_sess.h index 2ef28264..5e59aaab 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -29,7 +29,7 @@ struct tcpsrv_s; /* the tcps_sess object */ -typedef struct tcps_sess_s { +struct tcps_sess_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ tcpsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ tcpLstnPortList_t *pLstnInfo; /* pointer back to listener info */ @@ -46,8 +46,9 @@ typedef struct tcps_sess_s { uchar *pMsg; /* message (fragment) received */ uchar *fromHost; uchar *fromHostIP; - void *pUsr; /* a user-pointer */ -} tcps_sess_t; + void *pUsr; /* a user-pointer */ + rsRetVal (*DoSubmitMessage)(tcps_sess_t*, uchar*, int); /* submit message callback */ +}; /* interfaces */ @@ -67,6 +68,7 @@ BEGINinterface(tcps_sess) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetHostIP)(tcps_sess_t *pThis, uchar*); rsRetVal (*SetStrm)(tcps_sess_t *pThis, netstrm_t*); rsRetVal (*SetMsgIdx)(tcps_sess_t *pThis, int); + rsRetVal (*SetOnMsgReceive)(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); ENDinterface(tcps_sess) #define tcps_sessCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* interface changes diff --git a/tcpsrv.c b/tcpsrv.c index bbd95058..249eeecf 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -380,14 +380,15 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, errno = 0; errmsg.LogError(0, RS_RET_MAX_SESS_REACHED, "too many tcp sessions - dropping incoming request"); ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); - } else { - /* we found a free spot and can construct our session object */ - CHKiRet(tcps_sess.Construct(&pSess)); - CHKiRet(tcps_sess.SetTcpsrv(pSess, pThis)); - CHKiRet(tcps_sess.SetLstnInfo(pSess, pLstnInfo)); } - /* OK, we have a "good" index... */ + /* we found a free spot and can construct our session object */ + CHKiRet(tcps_sess.Construct(&pSess)); + CHKiRet(tcps_sess.SetTcpsrv(pSess, pThis)); + CHKiRet(tcps_sess.SetLstnInfo(pSess, pLstnInfo)); + if(pThis->OnMsgReceive != NULL) + CHKiRet(tcps_sess.SetOnMsgReceive(pSess, pThis->OnMsgReceive)); + /* get the host name */ CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); @@ -568,6 +569,7 @@ finalize_it: /* this is a very special case - this time only we do not exit the BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ pThis->iSessMax = TCPSESS_MAX_DEFAULT; /* TODO: useful default ;) */ pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + pThis->OnMsgReceive = NULL; ENDobjConstruct(tcpsrv) @@ -713,6 +715,16 @@ SetUsrP(tcpsrv_t *pThis, void *pUsr) RETiRet; } +static rsRetVal +SetOnMsgReceive(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)) +{ + DEFiRet; + assert(OnMsgReceive != NULL); + pThis->OnMsgReceive = OnMsgReceive; + RETiRet; +} + + /* Set additional framing to use (if any) -- rgerhards, 2008-12-10 */ static rsRetVal @@ -731,7 +743,6 @@ SetInputName(tcpsrv_t *pThis, uchar *name) { uchar *pszName; DEFiRet; -dbgprintf("XXX: SetInputName: %s\n", name); ISOBJ_TYPE_assert(pThis, tcpsrv); if(name == NULL) pszName = NULL; @@ -843,6 +854,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetCBOnDestruct = SetCBOnDestruct; pIf->SetCBOnRegularClose = SetCBOnRegularClose; pIf->SetCBOnErrClose = SetCBOnErrClose; + pIf->SetOnMsgReceive = SetOnMsgReceive; finalize_it: ENDobjQueryInterface(tcpsrv) diff --git a/tcpsrv.h b/tcpsrv.h index 0f7dd6c6..2d174ce0 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -71,6 +71,7 @@ struct tcpsrv_s { rsRetVal (*pOnSessAccept)(tcpsrv_t *, tcps_sess_t*); rsRetVal (*OnSessConstructFinalize)(void*); rsRetVal (*pOnSessDestruct)(void*); + rsRetVal (*OnMsgReceive)(tcps_sess_t *, uchar *pszMsg, int iLenMsg); /* submit message callback */ }; @@ -104,8 +105,10 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetCBOnSessConstructFinalize)(tcpsrv_t*, rsRetVal (*) (void*)); /* added v5 */ rsRetVal (*SetSessMax)(tcpsrv_t *pThis, int iMaxSess); /* 2009-04-09 */ + /* added v6 */ + rsRetVal (*SetOnMsgReceive)(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); /* 2009-05-24 */ ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 diff --git a/tests/Makefile.am b/tests/Makefile.am index c0e629d3..ed48fce8 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -46,6 +46,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12515 \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ + waitqueueempty.sh \ cfg.sh ourtail_SOURCES = ourtail.c diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 6384eb64..a91f3414 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -17,7 +17,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 4 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work diff --git a/tests/fieldtest.sh b/tests/fieldtest.sh index 7a646f00..482fa143 100755 --- a/tests/fieldtest.sh +++ b/tests/fieldtest.sh @@ -1,5 +1,5 @@ echo test fieldtest via udp -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tfield1 -iudp if [ "$?" -ne "0" ]; then diff --git a/tests/inputname.sh b/tests/inputname.sh index 7d9ea111..e1a58517 100755 --- a/tests/inputname.sh +++ b/tests/inputname.sh @@ -1,5 +1,5 @@ echo testing $InputTCPServerInputName directive -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason echo port 12514 ./nettester -tinputname_imtcp_12514 -cinputname_imtcp -itcp -p12514 diff --git a/tests/manytcp.sh b/tests/manytcp.sh index d9b2e9a0..06bd38b6 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -8,7 +8,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 5 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index 7b4b5611..2c2a8ef3 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,5 +1,5 @@ echo test omod-if-array via udp -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tomod-if-array -iudp -p4711 if [ "$?" -ne "0" ]; then diff --git a/tests/parsertest.sh b/tests/parsertest.sh index 152d8b60..afdb9469 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,5 +1,5 @@ echo test parsertest via udp -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tparse1 -iudp if [ "$?" -ne "0" ]; then diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf index 8851a459..017ee96d 100644 --- a/tests/testsuites/diskqueue.conf +++ b/tests/testsuites/diskqueue.conf @@ -4,6 +4,9 @@ $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + $ErrorMessagesToStderr off # set spool locations and switch queue to disk-only mode diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index 8175732e..3867da46 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -6,6 +6,9 @@ $MaxOpenFiles 2000 $InputTCPMaxSessions 1100 $InputTCPServerRun 13514 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + $ErrorMessagesToStderr off $template outfmt,"%msg:F,58:2%\n" diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh new file mode 100755 index 00000000..2c047588 --- /dev/null +++ b/tests/waitqueueempty.sh @@ -0,0 +1,4 @@ +# wait until main message queue is empty. This is currently done in +# a separate shell script so that we can change the implementation +# at some later point. -- rgerhards, 2009-05-25 +echo WaitMainQueueEmpty | nc 127.0.0.1 13500 diff --git a/tools/syslogd.c b/tools/syslogd.c index dc5b8fee..0b860448 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -350,10 +350,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bDebugPrintModuleList = 1; bEscapeCCOnRcv = 1; /* default is to escape control characters */ bReduceRepeatMsgs = 0; - if(pszMainMsgQFName != NULL) { - free(pszMainMsgQFName); - pszMainMsgQFName = NULL; - } + free(pszMainMsgQFName); + pszMainMsgQFName = NULL; iMainMsgQueueSize = 10000; iMainMsgQHighWtrMark = 8000; iMainMsgQLowWtrMark = 2000; @@ -412,6 +410,26 @@ static int usage(void) } +/* ------------------------------ some support functions for imdiag ------------------------------ * + * This is a bit dirty, but the only way to do it, at least with reasonable effort. + * rgerhards, 2009-05-25 + */ + +/* return back the approximate current number of messages in the main message queue + */ +rsRetVal +diagGetMainMsgQSize(int *piSize) +{ + DEFiRet; + assert(piSize != NULL); + *piSize = pMsgQueue->iQueueSize; + RETiRet; +} + + +/* ------------------------------ end support functions for imdiag ------------------------------ */ + + /* function to destruct a selector_t object * rgerhards, 2007-08-01 */ @@ -2658,7 +2676,6 @@ init(void) ABORT_FINALIZE(RS_RET_VALIDATION_RUN); /* switch the message object to threaded operation, if necessary */ -/* TODO:XXX: I think we must do this also if we have action queues! -- rgerhards, 2009-01-26 */ if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { MsgEnableThreadSafety(); } -- cgit From aaef9aa018dc030a7b5b2585bad19812ff214fab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 11:31:53 +0200 Subject: preparing for 4.3.1 --- ChangeLog | 2 +- configure.ac | 2 +- doc/how2help.html | 116 +++++++++++++++++++++++++++--------------------------- doc/manual.html | 2 +- 4 files changed, 60 insertions(+), 62 deletions(-) diff --git a/ChangeLog b/ChangeLog index a0357889..9d57f5e1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) - performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker diff --git a/configure.ac b/configure.ac index 8c8db8e1..66f1cab1 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.3.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.3.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/how2help.html b/doc/how2help.html index 0caa5a3a..4f0bd57a 100644 --- a/doc/how2help.html +++ b/doc/how2help.html @@ -1,59 +1,57 @@ - - -How you can Help - - -

      How you can Help

      -

      You like rsyslog and would like to lend us a helping hand? This page -tells you how easy it is to help a little bit. You can contribute to the project -even with a single mouse click! If you could pick a single item from the -wish list, that would be awfully helpful!

      -

      This is our wish list:

      -
        -
      • let others know how great rsyslog is -
        • -
      • let us know about rsyslog - we are eager for feedback
          -
        • tell us what you like and what you not like - so that we can include - that into development
          • -
        • tell us what you use rsyslog for - especially if you have high - traffic volume or an otherwise "uncommon" deployment. We are looking for - case studies and experience how rsyslog performs in unusual scenarios.
          • -
        • allow us to post your thoughts and experiences as a "user story" on - the web site (so far, none are there ;))
          • -
        -
        • -
      • if you know how to create packages (rpm, deb, ...)
          -
        • we would very much appreciate your help with package creation. We know - that it is important to have good binary packages for a product to - spread widely. Yet, we do not have the knowledge to do it all ourselves. - Drop Rainer a note if you - could help us out.
          • -
        -
        • -
      • if you have configured a device for sending syslog data, and that device - is not in our - syslog - configuration database, you might want to tell us how to configure it.
        • -
      • if you are a corporate user -
        • -
      -

      We appreciate your help very much. A big thank you for anything you -might do!

      - - - + + +How you can Help + + +

      How you can Help

      +

      You like rsyslog and would like to lend us a helping hand? This page +tells you how easy it is to help a little bit. You can contribute to the project +even with a single mouse click! If you could pick a single item from the +wish list, that would be awfully helpful!

      +

      This is our wish list:

      +
        +
      • let others know how great rsyslog is
          +
        • spread word about rsyslog in forums and newsgroups
          • +
        • place a link to www.rsyslog.com + from your home page
          • +
        +
        • +
      • let us know about rsyslog - we are eager for feedback
          +
        • tell us what you like and what you not like - so that we can include + that into development
          • +
        • tell us what you use rsyslog for - especially if you have high + traffic volume or an otherwise "uncommon" deployment. We are looking for + case studies and experience how rsyslog performs in unusual scenarios.
          • +
        • allow us to post your thoughts and experiences as a "user story" on + the web site (so far, none are there ;))
          • +
        +
        • +
      • if you know how to create packages (rpm, deb, ...)
          +
        • we would very much appreciate your help with package creation. We know + that it is important to have good binary packages for a product to + spread widely. Yet, we do not have the knowledge to do it all ourselves. + Drop Rainer a note if you + could help us out.
          • +
        +
        • +
      • if you have configured a device for sending syslog data, and that device + is not in our + syslog + configuration database, you might want to tell us how to configure it.
        • +
      • if you are a corporate user +
        • +
      +

      We appreciate your help very much. A big thank you for anything you +might do!

      + + + available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.3.0 (devel branch) of rsyslog. +

      This documentation is for version 4.3.1 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit From 7a7ec37f99f3dd5120952e6ca6263dd72061abb1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 13:02:06 +0200 Subject: improved testbench / solved imdiag race condition imdiag/imtcp had a modload race condition (as imdiag is a testing aid, this has no implications for production deployments). Also, I replaced netcat by a custom program to talk to imdiag. This, for the first time ever, is now a Java program. I plan to add some GUI troubleshooting tools and thought it is a good idea to start doing things in Java that can simply be done in that language. --- ChangeLog | 3 +++ Makefile.am | 2 +- runtime/modules.c | 21 +++++++++++++++++++++ tcps_sess.c | 23 ++++++++++++----------- tests/DiagTalker.java | 43 +++++++++++++++++++++++++++++++++++++++++++ tests/Makefile.am | 6 +++++- tests/diskqueue.sh | 4 ++++ tests/waitqueueempty.sh | 3 ++- 8 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 tests/DiagTalker.java diff --git a/ChangeLog b/ChangeLog index 9d57f5e1..2fc760c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 4.3.? [DEVEL] (rgerhards), 2009-??-?? +- bugfix: imdiag/imtcp had a race condition +--------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) - performance enhancement: imtcp calls parser no longer on input thread diff --git a/Makefile.am b/Makefile.am index 8d57700f..e991f323 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,5 +131,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave ACLOCAL_AMFLAGS = -I m4 diff --git a/runtime/modules.c b/runtime/modules.c index 9fdb48e7..3e8662a3 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -40,6 +40,7 @@ #include #include #include +#include #ifdef OS_BSD # include "libgen.h" #endif @@ -61,6 +62,14 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +/* we must ensure that only one thread at one time tries to load or unload + * modules, otherwise we may see race conditions. This first came up with + * imdiag/imtcp, which both use the same stream drivers. Below is the mutex + * for that handling. + * rgerhards, 2009-05-25 + */ +static pthread_mutex_t mutLoadUnload; + static modInfo_t *pLoadedModules = NULL; /* list of currently-loaded modules */ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ @@ -479,6 +488,8 @@ modUnlinkAndDestroy(modInfo_t **ppThis) pThis = *ppThis; assert(pThis != NULL); + pthread_mutex_lock(&mutLoadUnload); + /* first check if we are permitted to unload */ if(pThis->eType == eMOD_LIB) { if(pThis->uRefCnt > 0) { @@ -513,6 +524,7 @@ modUnlinkAndDestroy(modInfo_t **ppThis) moduleDestruct(pThis); finalize_it: + pthread_mutex_unlock(&mutLoadUnload); RETiRet; } @@ -587,6 +599,8 @@ Load(uchar *pModName) assert(pModName != NULL); dbgprintf("Requested to load module '%s'\n", pModName); + pthread_mutex_lock(&mutLoadUnload); + iModNameLen = strlen((char *) pModName); if(iModNameLen > 3 && !strcmp((char *) pModName + iModNameLen - 3, ".so")) { iModNameLen -= 3; @@ -696,6 +710,7 @@ Load(uchar *pModName) } finalize_it: + pthread_mutex_unlock(&mutLoadUnload); RETiRet; } @@ -791,6 +806,7 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + pthread_mutex_destroy(&mutLoadUnload); # ifdef DEBUG modUsrPrintAll(); /* debug aid - TODO: integrate with debug.c, at least the settings! */ @@ -833,6 +849,7 @@ ENDobjQueryInterface(module) */ BEGINAbstractObjClassInit(module, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE class also in END MACRO! */ uchar *pModPath; + pthread_mutexattr_t mutAttr; /* use any module load path specified in the environment */ if((pModPath = (uchar*) getenv("RSYSLOG_MODDIR")) != NULL) { @@ -850,6 +867,10 @@ BEGINAbstractObjClassInit(module, 1, OBJ_IS_CORE_MODULE) /* class, version - CHA SetModDir(glblModPath); } + pthread_mutexattr_init(&mutAttr); + pthread_mutexattr_settype(&mutAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&mutLoadUnload, &mutAttr); + /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); ENDObjClassInit(module) diff --git a/tcps_sess.c b/tcps_sess.c index c4548804..62d51f66 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -58,7 +58,6 @@ static int iMaxLine; /* maximum size of a single message */ /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); -static rsRetVal defaultDoSubmitMessage(tcps_sess_t *pThis, uchar*, int); /* Standard-Constructor */ @@ -66,7 +65,6 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->iMsg = 0; /* just make sure... */ pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ - pThis->DoSubmitMessage = defaultDoSubmitMessage; /* now allocate the message reception buffer */ CHKmalloc(pThis->pMsg = (uchar*) malloc(sizeof(uchar) * iMaxLine + 1)); finalize_it: @@ -228,11 +226,8 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) +defaultDoSubmitMessage(tcps_sess_t *pThis) { -// TODO: make calling this overridable so that the diag module can ask to be called -// and so it can do its work right in this entry point (but we need to check that -// we have the capability to send a reply at this point). msg_t *pMsg; struct syslogTime stTime; time_t ttGenTime; @@ -240,6 +235,11 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) ISOBJ_TYPE_assert(pThis, tcps_sess); + if(pThis->DoSubmitMessage != NULL) { + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + FINALIZE; + } + //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); //} @@ -247,7 +247,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); - memcpy(pMsg->pszRawMsg, pszMsg, iLenMsg); + memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); pMsg->iLenRawMsg = pThis->iMsg; MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); @@ -266,6 +266,7 @@ finalize_it: } + /* This should be called before a normal (non forced) close * of a TCP session. This function checks if there is any unprocessed * message left in the TCP stream. Such a message is probably a @@ -305,7 +306,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); } finalize_it: @@ -386,7 +387,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -397,7 +398,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -414,7 +415,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } } diff --git a/tests/DiagTalker.java b/tests/DiagTalker.java new file mode 100644 index 00000000..e33a5867 --- /dev/null +++ b/tests/DiagTalker.java @@ -0,0 +1,43 @@ +//package com.rsyslog.diag; +import java.io.*; +import java.net.*; + +public class DiagTalker { + public static void main(String[] args) throws IOException { + + Socket diagSocket = null; + PrintWriter out = null; + BufferedReader in = null; + final String host = "127.0.0.1"; + final int port = 13500; + + try { + diagSocket = new Socket(host, port); + out = new PrintWriter(diagSocket.getOutputStream(), true); + in = new BufferedReader(new InputStreamReader( + diagSocket.getInputStream())); + } catch (UnknownHostException e) { + System.err.println("can not resolve " + host + "!"); + System.exit(1); + } catch (IOException e) { + System.err.println("Couldn't get I/O for " + + "the connection to: " + host + "."); + System.exit(1); + } + + BufferedReader stdIn = new BufferedReader( + new InputStreamReader(System.in)); + String userInput; + + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + + out.close(); + in.close(); + stdIn.close(); + diagSocket.close(); + } +} + diff --git a/tests/Makefile.am b/tests/Makefile.am index ed48fce8..caa95c51 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -5,9 +5,12 @@ if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ -DISTCLEANFILES=rsyslog.pid +DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class test_files = testbench.h runtime-dummy.c +check_JAVA = DiagTalker.java +#dist_java_JAVA = DiagTalker.java + EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ @@ -47,6 +50,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ waitqueueempty.sh \ + DiagTalker.java \ cfg.sh ourtail_SOURCES = ourtail.c diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index a91f3414..eabfcf78 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -4,6 +4,10 @@ # memory to disk mode for DA queues. # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +#set -o xtrace +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="tmp" echo testing queue disk-only mode rm -rf test-spool mkdir test-spool diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh index 2c047588..4825853a 100755 --- a/tests/waitqueueempty.sh +++ b/tests/waitqueueempty.sh @@ -1,4 +1,5 @@ # wait until main message queue is empty. This is currently done in # a separate shell script so that we can change the implementation # at some later point. -- rgerhards, 2009-05-25 -echo WaitMainQueueEmpty | nc 127.0.0.1 13500 +#echo WaitMainQueueEmpty | nc 127.0.0.1 13500 +echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker -- cgit From b9549380270fa68e27e8ee3f049c7d34156a85ff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 14:16:31 +0200 Subject: solved some issues with testbench & a race condition --- runtime/modules.c | 10 +++++++++- tests/imtcp-multiport.sh | 6 +++--- tests/testsuites/imtcp-multiport.conf | 3 +++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/runtime/modules.c b/runtime/modules.c index 3e8662a3..32ae659f 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -806,7 +806,15 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); - pthread_mutex_destroy(&mutLoadUnload); + /* We have a problem in our reference counting, which leads to this function + * being called too early. This usually is no problem, but if we destroy + * the mutex object, we get into trouble. So rather than finding the root cause, + * we do not release the mutex right now and have a very, very slight leak. + * We know that otherwise no bad effects happen, so this acceptable for the + * time being. -- rgerhards, 2009-05-25 + * + * TODO: add again: pthread_mutex_destroy(&mutLoadUnload); + */ # ifdef DEBUG modUsrPrintAll(); /* debug aid - TODO: integrate with debug.c, at least the settings! */ diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh index aa1f15e8..17480dae 100755 --- a/tests/imtcp-multiport.sh +++ b/tests/imtcp-multiport.sh @@ -17,7 +17,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 2 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work @@ -42,7 +42,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 2 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work @@ -67,7 +67,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 2 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf index 00b63cb2..ec059fe4 100644 --- a/tests/testsuites/imtcp-multiport.conf +++ b/tests/testsuites/imtcp-multiport.conf @@ -6,6 +6,9 @@ $InputTCPServerRun 13514 $InputTCPServerRun 13515 $InputTCPServerRun 13516 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + $ErrorMessagesToStderr off $template outfmt,"%msg:F,58:2%\n" -- cgit From 598b1d73ca7815dc19906f86c6700402dc869eb4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 14:48:38 +0200 Subject: updated project status --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index c8608f73..4e8f1a5f 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,12 +2,12 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-04-21.

      +

      This page reflects the status as of 2009-05-25.

      Current Releases

      -

      development: 4.3.0 [2009-04-17] - -change log - -download +

      development: 4.3.1 [2009-05-25] - +change log - +download
      beta: 3.21.11 [2009-04-03] - change log - -- cgit From ddf74cc165c828139632170224356175c9c24e96 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 15:39:18 +0200 Subject: added test suite for persistent queue shutdown --- tests/Makefile.am | 5 ++++- tests/memq-persist.sh | 43 +++++++++++++++++++++++++++++++++++++ tests/testsuites/memq-persist1.conf | 25 +++++++++++++++++++++ tests/testsuites/memq-persist2.conf | 20 +++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100755 tests/memq-persist.sh create mode 100644 tests/testsuites/memq-persist1.conf create mode 100644 tests/testsuites/memq-persist2.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index caa95c51..92401511 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh memq-persist.sh if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif @@ -50,6 +50,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ waitqueueempty.sh \ + memq-persist.sh \ + testsuites/memq-persist1.sh \ + testsuites/memq-persist2.sh \ DiagTalker.java \ cfg.sh diff --git a/tests/memq-persist.sh b/tests/memq-persist.sh new file mode 100755 index 00000000..108cba57 --- /dev/null +++ b/tests/memq-persist.sh @@ -0,0 +1,43 @@ +# Test for memory queue which is persisted at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-25 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +#set -o xtrace +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="log" +echo testing memory queue persisting to disk +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason +rm -rf test-spool +mkdir test-spool +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist1.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# 20000 messages should be enough +./tcpflood 127.0.0.1 13514 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 3 # we need to wait to ensure everything is received (less 1 second would be better) +kill `cat rsyslog.pid` +sleep 5 # wait for engine to terminate +echo There must exist some files now: +ls -l test-spool +# restart engine and have rest processed +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist2.conf & +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +rm -rf test-spool diff --git a/tests/testsuites/memq-persist1.conf b/tests/testsuites/memq-persist1.conf new file mode 100644 index 00000000..5240090f --- /dev/null +++ b/tests/testsuites/memq-persist1.conf @@ -0,0 +1,25 @@ +# Test for persisting messages to disk on shutdown +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 1 +$MainMsgQueueSaveOnShutdown on +$InputTCPServerRun 13514 + +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + +$ModLoad ../plugins/omtesting/.libs/omtesting + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType LinkedList + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +# delay execution so that a queue can build up: +*.* :omtesting:sleep 0 1000 diff --git a/tests/testsuites/memq-persist2.conf b/tests/testsuites/memq-persist2.conf new file mode 100644 index 00000000..23e29e2f --- /dev/null +++ b/tests/testsuites/memq-persist2.conf @@ -0,0 +1,20 @@ +# Test for persisting messages to disk on shutdown +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$MainMsgQueueSaveOnShutdown on +$InputTCPServerRun 13514 + +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType LinkedList + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit From a900a7c34b674573f4b86350af0d68838da6550a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 May 2009 11:29:47 +0200 Subject: greatly enhanced testbench The imdiag module now can very effectively inject messages, which also frees us from uncertainties of tcp reception and processing. All shell script based tests have been modularized, what makes it far easier to create new tests. Also, the test bench now executes more reliable and much faster, because we can now rely on actual engine information where we previously did just a dumb sleep. --- ChangeLog | 1 + plugins/imdiag/imdiag.c | 145 +++++++++++++++++++++++++++++++--- plugins/imtcp/imtcp.c | 1 - runtime/netstrm.c | 1 - tcpsrv.c | 3 - tests/Makefile.am | 13 +-- tests/diag.sh | 82 +++++++++++++++++++ tests/diskqueue.sh | 29 ++----- tests/imtcp-multiport.sh | 78 +++++------------- tests/manytcp.sh | 27 ++----- tests/memq-persist.sh | 43 ---------- tests/queue-persist-drvr.sh | 28 +++++++ tests/queue-persist.sh | 11 +++ tests/testsuites/diag-common.conf | 16 ++++ tests/testsuites/diskqueue.conf | 7 +- tests/testsuites/imtcp-multiport.conf | 7 +- tests/testsuites/manytcp.conf | 7 +- tests/testsuites/memq-persist1.conf | 25 ------ tests/testsuites/memq-persist2.conf | 20 ----- tests/testsuites/queue-persist.conf | 23 ++++++ tests/waitqueueempty.sh | 5 -- 21 files changed, 337 insertions(+), 235 deletions(-) create mode 100755 tests/diag.sh delete mode 100755 tests/memq-persist.sh create mode 100755 tests/queue-persist-drvr.sh create mode 100755 tests/queue-persist.sh create mode 100644 tests/testsuites/diag-common.conf delete mode 100644 tests/testsuites/memq-persist1.conf delete mode 100644 tests/testsuites/memq-persist2.conf create mode 100644 tests/testsuites/queue-persist.conf delete mode 100755 tests/waitqueueempty.sh diff --git a/ChangeLog b/ChangeLog index 2fc760c3..d8fae145 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.? [DEVEL] (rgerhards), 2009-??-?? - bugfix: imdiag/imtcp had a race condition +- improved testbench (now much better code design and reuse) --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 40f94692..c700cab7 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -52,6 +52,8 @@ #include "errmsg.h" #include "tcpsrv.h" #include "srUtils.h" +#include "msg.h" +#include "datetime.h" #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT @@ -63,6 +65,7 @@ DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) /* Module static data */ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ @@ -134,10 +137,123 @@ onErrClose(tcps_sess_t *pSess) /* ------------------------------ end callbacks ------------------------------ */ +/* get the first word delimited by space from a given string. The pointer is + * advanced to after the word. Any leading spaces are discarded. If the + * output buffer is too small, parsing ends on buffer full condition. + * An empty buffer is returned if there is no more data inside the string. + * rgerhards, 2009-05-27 + */ +#define TO_LOWERCASE 1 +#define NO_MODIFY 0 +static void +getFirstWord(uchar **ppszSrc, uchar *pszBuf, size_t lenBuf, int options) +{ + uchar c; + uchar *pszSrc = *ppszSrc; + + while(*pszSrc && *pszSrc == ' ') + ++pszSrc; /* skip to first non-space */ + + while(*pszSrc && *pszSrc != ' ' && lenBuf > 1) { + c = *pszSrc++; + if(options & TO_LOWERCASE) + c = tolower(c); + *pszBuf++ = c; + lenBuf--; + } + + *pszBuf = '\0'; + *ppszSrc = pszSrc; +} + + +/* send a response back to the originator + * rgerhards, 2009-05-27 + */ +static rsRetVal __attribute__((format(printf, 2, 3))) +sendResponse(tcps_sess_t *pSess, char *fmt, ...) +{ + va_list ap; + ssize_t len; + uchar buf[1024]; + DEFiRet; + + va_start(ap, fmt); + len = vsnprintf((char*)buf, sizeof(buf), fmt, ap); + va_end(ap); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + +finalize_it: + RETiRet; +} + + +/* actually submit a message to the rsyslog core + */ +static rsRetVal +doInjectMsg(int iNum) +{ + uchar szMsg[1024]; + msg_t *pMsg; + struct syslogTime stTime; + time_t ttGenTime; + DEFiRet; + + snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum); + + datetime.getCurrTime(&stTime, &ttGenTime); + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg)); + pMsg->iLenRawMsg = ustrlen(szMsg); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag")); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->bParseHOSTNAME = 1; + MsgSetRcvFrom(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* TODO: way may use the real sender here... */ + CHKiRet(MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1"))); + CHKiRet(submitMsg(pMsg)); + +finalize_it: + RETiRet; +} + + +/* This function injects messages. Command format: + * injectmsg + * rgerhards, 2009-05-27 + */ +static rsRetVal +injectMsg(uchar *pszCmd, tcps_sess_t *pSess) +{ + uchar wordBuf[1024]; + int iFrom; + int nMsgs; + int i; + DEFiRet; + + /* we do not check errors here! */ + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); + iFrom = atoi((char*)wordBuf); + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); + nMsgs = atoi((char*)wordBuf); + + for(i = 0 ; i < nMsgs ; ++i) { + doInjectMsg(i + iFrom); + } + + CHKiRet(sendResponse(pSess, "messages injected\n")); + +finalize_it: + RETiRet; +} + + /* This function waits until the main queue is drained (size = 0) */ static rsRetVal -waitMainQEmpty(void) +waitMainQEmpty(tcps_sess_t *pSess) { int iMsgQueueSize; DEFiRet; @@ -148,21 +264,21 @@ waitMainQEmpty(void) CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); } + CHKiRet(sendResponse(pSess, "mainqueue empty\n")); + finalize_it: RETiRet; } - /* Function to handle received messages. This is our core function! * rgerhards, 2009-05-24 */ static rsRetVal OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) { - ssize_t len; int iMsgQueueSize; uchar *pszMsg; - uchar buf[1024]; + uchar cmdBuf[1024]; DEFiRet; assert(pSess != NULL); @@ -176,17 +292,18 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) memcpy(pszMsg, pRcv, iLenMsg); pszMsg[iLenMsg] = '\0'; - if(!ustrcmp(pszMsg, UCHAR_CONSTANT("GetMainMsgQueueSize"))) { + getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf)/sizeof(uchar), TO_LOWERCASE); + + if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "%d\n", iMsgQueueSize); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); - } else if(!ustrcmp(pszMsg, UCHAR_CONSTANT("WaitMainQueueEmpty"))) { - CHKiRet(waitMainQEmpty()); - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "mainqueue empty\n"); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); + } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) { + CHKiRet(waitMainQEmpty(pSess)); + } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) { + CHKiRet(injectMsg(pszMsg, pSess)); } else { - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "unkown command '%s'\n", pszMsg); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + dbgprintf("imdiag unkown command '%s'\n", cmdBuf); + CHKiRet(sendResponse(pSess, "unkown command '%s'\n", cmdBuf)); } finalize_it: @@ -285,6 +402,7 @@ CODESTARTmodExit objRelease(tcps_sess, LM_TCPSRV_FILENAME); objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); ENDmodExit @@ -321,6 +439,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverrun"), 0, eCmdHdlrGetWord, diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index f0efe1ee..84e660bc 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -180,7 +180,6 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa } } -dbgprintf("XXX: try add listen port %s\n", pNewVal); /* initialized, now add socket */ CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? UCHAR_CONSTANT("imtcp") : pszInputName)); diff --git a/runtime/netstrm.c b/runtime/netstrm.c index f6a8de7f..05bb25c0 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -114,7 +114,6 @@ AcceptConnReq(netstrm_t *pThis, netstrm_t **ppNew) ISOBJ_TYPE_assert(pThis, netstrm); assert(ppNew != NULL); -RUNLOG_STR("XXX: accept conn reqeust"); /* accept the new connection */ CHKiRet(pThis->Drvr.AcceptConnReq(pThis->pDrvrData, &pNewNsd)); /* construct our object so that we can use it... */ diff --git a/tcpsrv.c b/tcpsrv.c index 249eeecf..02eee88e 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -97,7 +97,6 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) ISOBJ_TYPE_assert(pThis, tcpsrv); -dbgprintf("XXX: tcpsrv.c add port %s, name '%s'\n", pszPort, pThis->pszInputName); /* create entry */ CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; @@ -267,7 +266,6 @@ addTcpLstn(void *pUsr, netstrm_t *pLstn) tcpsrv_t *pThis = pPortList->pSrv; DEFiRet; -dbgprintf("XXX: addTcpLst name %s\n", pPortList->pszInputName); ISOBJ_TYPE_assert(pThis, tcpsrv); ISOBJ_TYPE_assert(pLstn, netstrm); @@ -326,7 +324,6 @@ create_tcp_socket(tcpsrv_t *pThis) /* init all configured ports */ pEntry = pThis->pLstnPorts; while(pEntry != NULL) { -dbgprintf("XXX: tcpsrv.c create_tcp_socket do port %s\n", pEntry->pszPort); CHKiRet(initTCPListener(pThis, pEntry)); pEntry = pEntry->pNext; } diff --git a/tests/Makefile.am b/tests/Makefile.am index 92401511..a2e6c34c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh memq-persist.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh queue-persist.sh if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif @@ -9,7 +9,6 @@ DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class test_files = testbench.h runtime-dummy.c check_JAVA = DiagTalker.java -#dist_java_JAVA = DiagTalker.java EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ @@ -34,6 +33,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.omod-if-array \ killrsyslog.sh \ parsertest.sh \ + queue-persist.sh \ + testsuites/queue-persist.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ imtcp-multiport.sh \ @@ -49,10 +50,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12515 \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ - waitqueueempty.sh \ - memq-persist.sh \ - testsuites/memq-persist1.sh \ - testsuites/memq-persist2.sh \ + diag.sh \ + queue-persist.sh \ + queue-persist-drvr.sh \ + testsuites/queue-persist.sh \ DiagTalker.java \ cfg.sh diff --git a/tests/diag.sh b/tests/diag.sh new file mode 100755 index 00000000..270b30d6 --- /dev/null +++ b/tests/diag.sh @@ -0,0 +1,82 @@ +# this shell script provides commands to the common diag system. It enables +# test scripts to wait for certain conditions and initiate certain actions. +# needs support in config file. +# NOTE: this file should be included with "source diag.sh", as it otherwise is +# not always able to convey back states to the upper-level test driver +# begun 2009-05-27 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +#set -o xtrace +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="tmp" +case $1 in + 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + rm -f rsyslogd.started work-*.conf + rm -f work rsyslog.out.log rsyslog.out.log.save # common work files + rm -rf test-spool + mkdir test-spool + ;; + 'exit') rm -f rsyslogd.started work-*.conf + rm -f work rsyslog.out.log rsyslog.out.log.save # common work files + rm -rf test-spool + ;; + 'startup') # start rsyslogd with default params. $2 is the config file name to use + # returns only after successful startup + ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $srcdir/diag.sh wait-startup + ;; + 'wait-startup') # wait for rsyslogd startup + while test ! -f rsyslogd.started; do + true + done + echo "rsyslogd started with pid " `cat rsyslog.pid` + ;; + 'wait-shutdown') # actually, we wait for rsyslog.pid to be deleted + while test -f rsyslog.pid; do + true + done + ;; + 'wait-queueempty') # wait for main message queue to be empty + echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker + ;; + 'shutdown-when-empty') # shut rsyslogd down when main queue is empty + $srcdir/diag.sh wait-queueempty + kill `cat rsyslog.pid` + # note: we do not wait for the actual termination! + ;; + 'shutdown-immediate') # shut rsyslogd down without emptying the queue + kill `cat rsyslog.pid` + # note: we do not wait for the actual termination! + ;; + 'tcpflood') # do a tcpflood run and check if it worked params are passed to tcpflood + ./tcpflood $2 $3 $4 $5 $6 $7 $8 + if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save + exit 1 + fi + ;; + 'injectmsg') # inject messages via our inject interface (imdiag) + echo injectmsg $2 $3 $4 $5 | java -classpath $abs_top_builddir DiagTalker + # TODO: some return state checking? (does it really make sense here?) + ;; + 'check-mainq-spool') # check if mainqueue spool files exist, if not abort (we just check .qi) + echo There must exist some files now: + ls -l test-spool + if test ! -f test-spool/mainq.qi; then + echo "error: mainq.qi does not exist where expected to do so!" + ls -l test-spool + exit 1 + fi + ;; + 'seq-check') # do the usual sequence check to see if everything was properly received + rm -f work + sort < rsyslog.out.log > work + ./chkseq work $2 $3 + if [ "$?" -ne "0" ]; then + rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 + fi + ;; + *) echo "invalid argument" $1 +esac diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index eabfcf78..bf1a46fd 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -9,27 +9,10 @@ #export RSYSLOG_DEBUG="debug nostdout" #export RSYSLOG_DEBUGLOG="tmp" echo testing queue disk-only mode -rm -rf test-spool -mkdir test-spool -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +source $srcdir/diag.sh init +source $srcdir/diag.sh startup diskqueue.conf # 20000 messages should be enough - the disk test is slow enough ;) -./tcpflood 127.0.0.1 13514 1 20000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 19999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log -rm -rf test-spool +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 19999 +source $srcdir/diag.sh exit diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh index 17480dae..702f8834 100755 --- a/tests/imtcp-multiport.sh +++ b/tests/imtcp-multiport.sh @@ -8,73 +8,31 @@ # added 2009-05-22 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 echo testing imtcp multiple listeners -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13514 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit # # # ### now complete new cycle with other port ### # # -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13515 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13515 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit # # # ### now complete new cycle with other port ### # # -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13516 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13516 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 06bd38b6..c55eb9c0 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,21 +1,8 @@ -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +# test many concurrent tcp connections +source $srcdir/diag.sh init +source $srcdir/diag.sh startup manytcp.conf # the config file specifies exactly 1100 connections -./tcpflood 127.0.0.1 13514 1000 40000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 39999 -if [ "$?" -ne "0" ]; then - rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1000 40000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/memq-persist.sh b/tests/memq-persist.sh deleted file mode 100755 index 108cba57..00000000 --- a/tests/memq-persist.sh +++ /dev/null @@ -1,43 +0,0 @@ -# Test for memory queue which is persisted at shutdown. The -# plan is to start an instance, emit some data, do a relatively -# fast shutdown and then re-start the engine to process the -# remaining data. -# added 2009-05-25 by Rgerhards -# This file is part of the rsyslog project, released under GPLv3 -# uncomment for debugging support: -#set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="log" -echo testing memory queue persisting to disk -$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason -rm -rf test-spool -mkdir test-spool -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist1.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -# 20000 messages should be enough -./tcpflood 127.0.0.1 13514 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -sleep 3 # we need to wait to ensure everything is received (less 1 second would be better) -kill `cat rsyslog.pid` -sleep 5 # wait for engine to terminate -echo There must exist some files now: -ls -l test-spool -# restart engine and have rest processed -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist2.conf & -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log -rm -rf test-spool diff --git a/tests/queue-persist-drvr.sh b/tests/queue-persist-drvr.sh new file mode 100755 index 00000000..ea5386a7 --- /dev/null +++ b/tests/queue-persist-drvr.sh @@ -0,0 +1,28 @@ +# Test for queue data persisting at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo testing memory queue persisting to disk, mode $1 +source $srcdir/diag.sh init + +# prepare config +echo \$MainMsgQueueType $1 > work-queuemode.conf +echo "*.* :omtesting:sleep 0 1000" > work-delay.conf + +# inject 5000 msgs, so that we do not hit the high watermark +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh injectmsg 0 5000 +$srcdir/diag.sh shutdown-immediate +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh check-mainq-spool + +# restart engine and have rest processed +#remove delay +echo "#" > work-delay.conf +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/queue-persist.sh b/tests/queue-persist.sh new file mode 100755 index 00000000..999655b1 --- /dev/null +++ b/tests/queue-persist.sh @@ -0,0 +1,11 @@ +# Test for queue data persisting at shutdown. We use the actual driver +# to carry out multiple tests with different queue modes +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +source $srcdir/queue-persist-drvr.sh LinkedList +source $srcdir/queue-persist-drvr.sh FixedArray +# the disk test should not fail, however, the config is extreme and using +# it more or less is a config error +source $srcdir/queue-persist-drvr.sh Disk +# we do not test Direct mode because this absolute can not work in direct mode +# (maybe we should do a fail-type of test?) diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf new file mode 100644 index 00000000..9e9e28fe --- /dev/null +++ b/tests/testsuites/diag-common.conf @@ -0,0 +1,16 @@ +# This is a config include file. It sets up rsyslog so that the +# diag system can successfully be used. Also, it generates a file +# "rsyslogd.started" after rsyslogd is initialized. This config file +# should be included in all tests that intend to use common code for +# controlling the daemon. +# NOTE: we assume that rsyslogd's current working directory is +# ./tests (or the distcheck equivalent), in particlular that this +# config file resides in the testsuites subdirectory. +# rgerhards, 2009-05-27 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + +$template startupfile,"rsyslogd.started" # trick to use relative path names! +:syslogtag, contains, "rsyslogd" ?startupfile + +$ErrorMessagesToStderr off diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf index 017ee96d..d7f323bc 100644 --- a/tests/testsuites/diskqueue.conf +++ b/tests/testsuites/diskqueue.conf @@ -1,14 +1,11 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-04-17 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - # set spool locations and switch queue to disk-only mode $WorkDirectory test-spool $MainMsgQueueFilename mainq diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf index ec059fe4..9146f6e0 100644 --- a/tests/testsuites/imtcp-multiport.conf +++ b/tests/testsuites/imtcp-multiport.conf @@ -1,16 +1,13 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-05-22 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 $InputTCPServerRun 13515 $InputTCPServerRun 13516 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! :msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index 3867da46..772ec9ce 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,16 +1,13 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $MaxOpenFiles 2000 $InputTCPMaxSessions 1100 $InputTCPServerRun 13514 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! :msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/memq-persist1.conf b/tests/testsuites/memq-persist1.conf deleted file mode 100644 index 5240090f..00000000 --- a/tests/testsuites/memq-persist1.conf +++ /dev/null @@ -1,25 +0,0 @@ -# Test for persisting messages to disk on shutdown -# rgerhards, 2009-04-17 -$ModLoad ../plugins/imtcp/.libs/imtcp -$MainMsgQueueTimeoutShutdown 1 -$MainMsgQueueSaveOnShutdown on -$InputTCPServerRun 13514 - -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ModLoad ../plugins/omtesting/.libs/omtesting - -$ErrorMessagesToStderr off - -# set spool locations and switch queue to disk-only mode -$WorkDirectory test-spool -$MainMsgQueueFilename mainq -$MainMsgQueueType LinkedList - -$template outfmt,"%msg:F,58:2%\n" -$template dynfile,"rsyslog.out.log" # trick to use relative path names! -:msg, contains, "msgnum:" ?dynfile;outfmt - -# delay execution so that a queue can build up: -*.* :omtesting:sleep 0 1000 diff --git a/tests/testsuites/memq-persist2.conf b/tests/testsuites/memq-persist2.conf deleted file mode 100644 index 23e29e2f..00000000 --- a/tests/testsuites/memq-persist2.conf +++ /dev/null @@ -1,20 +0,0 @@ -# Test for persisting messages to disk on shutdown -# rgerhards, 2009-04-17 -$ModLoad ../plugins/imtcp/.libs/imtcp -$MainMsgQueueTimeoutShutdown 10000 -$MainMsgQueueSaveOnShutdown on -$InputTCPServerRun 13514 - -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - -# set spool locations and switch queue to disk-only mode -$WorkDirectory test-spool -$MainMsgQueueFilename mainq -$MainMsgQueueType LinkedList - -$template outfmt,"%msg:F,58:2%\n" -$template dynfile,"rsyslog.out.log" # trick to use relative path names! -:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf new file mode 100644 index 00000000..80f8ba30 --- /dev/null +++ b/tests/testsuites/queue-persist.conf @@ -0,0 +1,23 @@ +# Test for persisting messages on shutdown +# rgerhards, 2009-04-17 +$IncludeConfig testsuites/diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 1 +$MainMsgQueueSaveOnShutdown on +$InputTCPServerRun 13514 + +$ModLoad ../plugins/omtesting/.libs/omtesting + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$IncludeConfig work-queuemode.conf + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +$IncludeConfig work-delay.conf diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh deleted file mode 100755 index 4825853a..00000000 --- a/tests/waitqueueempty.sh +++ /dev/null @@ -1,5 +0,0 @@ -# wait until main message queue is empty. This is currently done in -# a separate shell script so that we can change the implementation -# at some later point. -- rgerhards, 2009-05-25 -#echo WaitMainQueueEmpty | nc 127.0.0.1 13500 -echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker -- cgit From 74ed599ea6ab96cffba35ca14f5654523f89bc61 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 May 2009 11:36:54 +0200 Subject: fixed typo --- tests/Makefile.am | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index a2e6c34c..713c721e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -33,8 +33,6 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.omod-if-array \ killrsyslog.sh \ parsertest.sh \ - queue-persist.sh \ - testsuites/queue-persist.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ imtcp-multiport.sh \ @@ -53,7 +51,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ diag.sh \ queue-persist.sh \ queue-persist-drvr.sh \ - testsuites/queue-persist.sh \ + testsuites/queue-persist.conf \ DiagTalker.java \ cfg.sh -- cgit From 5514acb7eeaca41f1fd196402ef08efc55bce11c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 15:19:03 +0200 Subject: fixed make distcheck --- tests/Makefile.am | 1 + tests/diag.sh | 5 +++-- tests/testsuites/diskqueue.conf | 2 +- tests/testsuites/imtcp-multiport.conf | 2 +- tests/testsuites/manytcp.conf | 2 +- tests/testsuites/queue-persist.conf | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 713c721e..7a7b8147 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -49,6 +49,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ + testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ testsuites/queue-persist.conf \ diff --git a/tests/diag.sh b/tests/diag.sh index 270b30d6..ec2c1190 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -7,15 +7,16 @@ # This file is part of the rsyslog project, released under GPLv3 #set -o xtrace #export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="tmp" +#export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + cp $srcdir/testsuites/diag-common.conf diag-common.conf rm -f rsyslogd.started work-*.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files rm -rf test-spool mkdir test-spool ;; - 'exit') rm -f rsyslogd.started work-*.conf + 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files rm -rf test-spool ;; diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf index d7f323bc..a992c5a5 100644 --- a/tests/testsuites/diskqueue.conf +++ b/tests/testsuites/diskqueue.conf @@ -1,6 +1,6 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-04-17 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf index 9146f6e0..ccdc15fb 100644 --- a/tests/testsuites/imtcp-multiport.conf +++ b/tests/testsuites/imtcp-multiport.conf @@ -1,6 +1,6 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-05-22 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index 772ec9ce..eb9db257 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,6 +1,6 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf index 80f8ba30..81ee1be5 100644 --- a/tests/testsuites/queue-persist.conf +++ b/tests/testsuites/queue-persist.conf @@ -1,6 +1,6 @@ # Test for persisting messages on shutdown # rgerhards, 2009-04-17 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 1 -- cgit From c6a2dafb2fc5d186ef146035f270a4819075eab1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 16:30:57 +0200 Subject: doc: added info on when $OptimizeForUniprocessor was made available --- doc/rsyslog_conf_global.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 43eacc43..fe76912d 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -196,7 +196,7 @@ large enough for the whole message. (Introduced with 4.1.5). Once set, it affect

    • $ResetConfigVariables
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The -default may change as uniprocessor systems become less common.
      • +default may change as uniprocessor systems become less common. [available since 4.1.0]
    • $PreserveFQDN [on/off) - if set to off (legacy default to remain compatible to sysklogd), the domain part from a name that is within the same domain as the receiving system is stripped. If set to on, full names are always used.
      • -- cgit From 8e180e152ab1cd332a212648d6a0f2e2583a2bca Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 29 May 2009 12:06:59 +0200 Subject: added forgotten file --- doc/rsyslog-vers.png | Bin 0 -> 162637 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/rsyslog-vers.png diff --git a/doc/rsyslog-vers.png b/doc/rsyslog-vers.png new file mode 100644 index 00000000..e8ec8b81 Binary files /dev/null and b/doc/rsyslog-vers.png differ -- cgit From eafafc63597a600fceda5ac9ada90acd6e2f037e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 29 May 2009 14:31:50 +0200 Subject: fix: import from beta accidently disable imdiag --- plugins/imdiag/imdiag.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 129241fa..c700cab7 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -1,5 +1,3 @@ -#warning "imdiag is NOT supported in this version of rsyslog" -#if 0 /* imdiag.c * This is a diagnostics module, primarily meant for troubleshooting * and information about the runtime state of rsyslog. It is implemented @@ -459,7 +457,6 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit -#endif /* vim:set ai: -- cgit From 36fe8d92f8bad4ddc1f24ede14405129ec46c0d8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 29 May 2009 17:25:16 +0200 Subject: added ability for the UDP output action to rebind its send socket after sending n messages New config directive $ActionSendUDPRebindInterval added for the purpose. By default, rebinding is disabled. This is considered useful for load balancers. --- ChangeLog | 6 +++- doc/rsyslog_conf_global.html | 3 ++ runtime/conf.c | 3 +- tools/omfwd.c | 69 ++++++++++++++++++++++++++++++++------------ 4 files changed, 60 insertions(+), 21 deletions(-) diff --git a/ChangeLog b/ChangeLog index 931f21f8..832d7a6c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- -Version 4.3.? [DEVEL] (rgerhards), 2009-??-?? +Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- added ability for the UDP output action to rebind its send socket after + sending n messages. New config directive $ActionSendUDPRebindInterval + added for the purpose. By default, rebinding is disabled. This is + considered useful for load balancers. - bugfix: imdiag/imtcp had a race condition - improved testbench (now much better code design and reuse) --------------------------------------------------------------------------- diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index fe76912d..778e18f8 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -96,6 +96,9 @@ default 60000 (1 minute)] (driver-specific)
    • $ActionSendStreamDriverAuthMode <mode>,  authentication mode to use with the stream driver (driver-specific)
    • $ActionSendStreamDriverPermittedPeer <ID>,  accepted fingerprint (SHA1) or name of remote peer (driver-specific) - directive may go away!
      • +
    • $ActionSendUDPRebindInterval nbr- [available since 4.3.2] - instructs the UDP send +action to rebind the send socket every nbr of messages sent. Zero, the default, means +that no rebind is done. This directive is useful for use with load-balancers.
    • $AllowedSender
    • $ControlCharacterEscapePrefix
    • $DebugPrintCFSyslineHandlerList
      • diff --git a/runtime/conf.c b/runtime/conf.c index 602a5eb1..bb1fa282 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -484,7 +484,8 @@ finalize_it: } if(bHadAnError && (iRet == RS_RET_OK)) { /* a bit dirty, enhance in future releases */ - iRet = RS_RET_ERR; +// TODO: fix regression! + // iRet = RS_RET_ERR; } RETiRet; } diff --git a/tools/omfwd.c b/tools/omfwd.c index 88a382e0..c8fedfc9 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -82,12 +82,14 @@ typedef struct _instanceData { permittedPeers_t *pPermPeers; int iStrmDrvrMode; char *f_hname; - int *pSockArray; /* sockets to use for UDP */ + int *pSockArray; /* sockets to use for UDP */ int bIsConnected; /* are we connected to remote host? 0 - no, 1 - yes, UDP means addr resolved */ struct addrinfo *f_addr; - int compressionLevel; /* 0 - no compression, else level for zlib */ + int compressionLevel; /* 0 - no compression, else level for zlib */ char *port; int protocol; + int iUDPRebindInterval; /* rebind interval */ + int nXmit; /* number of transmissions since last (re-)bind */ # define FORW_UDP 0 # define FORW_TCP 1 /* following fields for TCP-based delivery */ @@ -100,9 +102,31 @@ static uchar *pszStrmDrvr = NULL; /* name of the stream driver to use */ static short iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ static short bResendLastOnRecon = 0; /* should the last message be re-sent on a successful reconnect? */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ +static int iUDPRebindInterval = 0; /* support for automatic re-binding (load balancers!). 0 - no rebind */ static permittedPeers_t *pPermPeers = NULL; +static rsRetVal doTryResume(instanceData *pData); + +/* Close the UDP sockets. + * rgerhards, 2009-05-29 + */ +static rsRetVal +closeUDPSockets(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + if(pData->pSockArray != NULL) { + net.closeUDPListenSockets(pData->pSockArray); + pData->pSockArray = NULL; + freeaddrinfo(pData->f_addr); + pData->f_addr = NULL; + } +pData->bIsConnected = 0; // TODO: remove this variable altogether + RETiRet; +} + + /* get the syslog forward port from selector_t. The passed in * struct must be one that is setup for forwarding. * rgerhards, 2007-06-28 @@ -148,30 +172,19 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - if(pData->f_addr != NULL) { /* TODO: is the check ok? */ - freeaddrinfo(pData->f_addr); - pData->f_addr = NULL; - } - if(pData->port != NULL) - free(pData->port); - /* final cleanup */ DestructTCPInstanceData(pData); - if(pData->pSockArray != NULL) - net.closeUDPListenSockets(pData->pSockArray); + closeUDPSockets(pData); if(pData->protocol == FORW_TCP) { tcpclt.Destruct(&pData->pTCPClt); } - if(pData->f_hname != NULL) - free(pData->f_hname); - if(pData->pszStrmDrvr != NULL) - free(pData->pszStrmDrvr); - if(pData->pszStrmDrvrAuthMode != NULL) - free(pData->pszStrmDrvrAuthMode); - if(pData->pPermPeers != NULL) - net.DestructPermittedPeers(&pData->pPermPeers); + free(pData->port); + free(pData->f_hname); + free(pData->pszStrmDrvr); + free(pData->pszStrmDrvrAuthMode); + net.DestructPermittedPeers(&pData->pPermPeers); ENDfreeInstance @@ -192,6 +205,18 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) unsigned lsent = 0; int bSendSuccess; +dbgprintf("rebind logic: interval %d, curr %d, mod %d, if %d\n", pData->iUDPRebindInterval, pData->nXmit, + (pData->nXmit % pData->iUDPRebindInterval), ((pData->nXmit % pData->iUDPRebindInterval) == 0)); + if(pData->iUDPRebindInterval && (pData->nXmit++ % pData->iUDPRebindInterval == 0)) { + dbgprintf("omfwd dropping UDP 'connection' (as configured)\n"); + pData->nXmit = 1; /* else we have an addtl wrap at 2^31-1 */ + CHKiRet(closeUDPSockets(pData)); + } + + if(pData->pSockArray == NULL) { + CHKiRet(doTryResume(pData)); + } + if(pData->pSockArray != NULL) { /* we need to track if we have success sending to the remote * peer. Success is indicated by at least one sendto() call @@ -224,6 +249,7 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) } } +finalize_it: RETiRet; } @@ -616,6 +642,9 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) CHKmalloc(pData->f_hname = strdup((char*) q)); } + /* copy over config data as needed */ + pData->iUDPRebindInterval = iUDPRebindInterval; + /* process template */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_TraditionalForwardFormat" : pszTplName)); @@ -699,6 +728,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a /* we now must reset all non-string values */ iStrmDrvrMode = 0; bResendLastOnRecon = 0; + iUDPRebindInterval = 0; return RS_RET_OK; } @@ -713,6 +743,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(net,LM_NET_FILENAME)); CHKiRet(regCfSysLineHdlr((uchar *)"actionforwarddefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszTplName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionsendudprebindinterval", 0, eCmdHdlrInt, NULL, &iUDPRebindInterval, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszStrmDrvr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdrivermode", 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdriverauthmode", 0, eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, NULL)); -- cgit From 25daa04902f2754a487973c4de707869d66b84c1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 07:44:07 +0200 Subject: added support for custom modules ... this provides some basic support to integrate extensions that are not direct parts of rsyslog to be built during its build process. --- Makefile.am | 4 ++++ configure.ac | 16 ++++++++++++++++ plugins/cust1/Makefile.am | 6 ++++++ 3 files changed, 26 insertions(+) create mode 100644 plugins/cust1/Makefile.am diff --git a/Makefile.am b/Makefile.am index e991f323..8a130655 100644 --- a/Makefile.am +++ b/Makefile.am @@ -84,6 +84,10 @@ if ENABLE_SNMP SUBDIRS += plugins/omsnmp endif +if ENABLE_CUST1 +SUBDIRS += plugins/cust1 +endif + if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif diff --git a/configure.ac b/configure.ac index 7150d211..9fc59f76 100644 --- a/configure.ac +++ b/configure.ac @@ -716,6 +716,20 @@ AC_ARG_ENABLE(omstdout, ) AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) +# This provides a vehicle to integrate custom modules, that are not +# part of rsyslog, into the build process. It is named cust1, so that +# additional such modules can easily be added. +AC_ARG_ENABLE(cust1, + [AS_HELP_STRING([--enable-cust1],[Compiles stdout module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_cust1="yes" ;; + no) enable_cust1="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-cust1) ;; + esac], + [enable_cust1=no] +) +AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes) + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate @@ -789,6 +803,7 @@ AC_CONFIG_FILES([Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ + plugins/cust1/Makefile \ tests/Makefile]) AC_OUTPUT @@ -802,6 +817,7 @@ echo " Regular expressions support enabled: $enable_regexp" echo " Zlib compression support enabled: $enable_zlib" echo " rsyslog runtime will be built: $enable_rsyslogrt" echo " rsyslogd will be built: $enable_rsyslogd" +echo " custom module 1 will be built: $enable_cust1" echo echo "---{ input plugins }---" echo " Klog functionality enabled: $enable_klog ($os_type)" diff --git a/plugins/cust1/Makefile.am b/plugins/cust1/Makefile.am new file mode 100644 index 00000000..d2e075f9 --- /dev/null +++ b/plugins/cust1/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = cust1.la + +cust1_la_SOURCES = cust1.c +cust1_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +cust1_la_LDFLAGS = -module -avoid-version +cust1_la_LIBADD = -- cgit From 14d1209640e3554c4284eab136f9932cfc441126 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 10:31:42 +0200 Subject: added a generic network stream server (in addition to rather specific syslog tcp server) --- ChangeLog | 2 + runtime/Makefile.am | 7 + runtime/rsyslog.h | 3 + runtime/strms_sess.c | 292 ++++++++++++++++ runtime/strms_sess.h | 74 ++++ runtime/strmsrv.c | 952 +++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/strmsrv.h | 110 ++++++ tcpsrv.c | 5 +- 8 files changed, 1444 insertions(+), 1 deletion(-) create mode 100644 runtime/strms_sess.c create mode 100644 runtime/strms_sess.h create mode 100644 runtime/strmsrv.c create mode 100644 runtime/strmsrv.h diff --git a/ChangeLog b/ChangeLog index 832d7a6c..ec971916 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- added a generic network stream server (in addition to rather specific + syslog tcp server) - added ability for the UDP output action to rebind its send socket after sending n messages. New config directive $ActionSendUDPRebindInterval added for the purpose. By default, rebinding is disabled. This is diff --git a/runtime/Makefile.am b/runtime/Makefile.am index bc03c4a7..eeb656d6 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -121,6 +121,13 @@ lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version lmnetstrms_la_LIBADD = +# generic stream server framework +pkglib_LTLIBRARIES += lmstrmsrv.la +lmstrmsrv_la_SOURCES = strmsrv.c strmsrv.h strms_sess.c strms_sess.h +lmstrmsrv_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmstrmsrv_la_LDFLAGS = -module -avoid-version +lmstrmsrv_la_LIBADD = + # netstream drivers # plain tcp driver - main driver diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 2f15f926..293e559b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -85,10 +85,13 @@ typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; typedef struct tcps_sess_s tcps_sess_t; +typedef struct strmsrv_s strmsrv_t; +typedef struct strms_sess_s strms_sess_t; typedef struct vmstk_s vmstk_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? +typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? /* some universal 64 bit define... */ typedef long long int64; diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c new file mode 100644 index 00000000..20920430 --- /dev/null +++ b/runtime/strms_sess.c @@ -0,0 +1,292 @@ +/* strms_sess.c + * + * This implements a session of the strmsrv object. For general + * comments, see header of strmsrv.c. + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include + +#include "rsyslog.h" +#include "dirty.h" +#include "module-template.h" +#include "net.h" +#include "strmsrv.h" +#include "strms_sess.h" +#include "obj.h" +#include "errmsg.h" +#include "netstrm.h" +#include "msg.h" +#include "datetime.h" + + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(datetime) + +static int iMaxLine; /* maximum size of a single message */ + +/* forward definitions */ +static rsRetVal Close(strms_sess_t *pThis); + + +/* Standard-Constructor */ +BEGINobjConstruct(strms_sess) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(strms_sess) + + +/* ConstructionFinalizer + */ +static rsRetVal +strms_sessConstructFinalize(strms_sess_t __attribute__((unused)) *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + if(pThis->pSrv->OnSessConstructFinalize != NULL) { + CHKiRet(pThis->pSrv->OnSessConstructFinalize(&pThis->pUsr)); + } + +finalize_it: + RETiRet; +} + + +/* destructor for the strms_sess object */ +BEGINobjDestruct(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strms_sess) + if(pThis->pStrm != NULL) + netstrm.Destruct(&pThis->pStrm); + + if(pThis->pSrv->pOnSessDestruct != NULL) { + pThis->pSrv->pOnSessDestruct(&pThis->pUsr); + } + /* now destruct our own properties */ + free(pThis->fromHost); + free(pThis->fromHostIP); +ENDobjDestruct(strms_sess) + + +/* debugprint for the strms_sess object */ +BEGINobjDebugPrint(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strms_sess) +ENDobjDebugPrint(strms_sess) + + +/* set property functions */ +/* set the hostname. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHost() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-04-24 + */ +static rsRetVal +SetHost(strms_sess_t *pThis, uchar *pszHost) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHost); + pThis->fromHost = pszHost; + RETiRet; +} + +/* set the remote host's IP. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHostIP() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-05-16 + */ +static rsRetVal +SetHostIP(strms_sess_t *pThis, uchar *pszHostIP) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHostIP); + pThis->fromHostIP = pszHostIP; + RETiRet; +} + +static rsRetVal +SetStrm(strms_sess_t *pThis, netstrm_t *pStrm) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + pThis->pStrm = pStrm; + RETiRet; +} + + +/* set our parent, the strmsrv object */ +static rsRetVal +SetStrmsrv(strms_sess_t *pThis, strmsrv_t *pSrv) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + ISOBJ_TYPE_assert(pSrv, strmsrv); + pThis->pSrv = pSrv; + RETiRet; +} + + +/* set our parent listener info*/ +static rsRetVal +SetLstnInfo(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pLstnInfo != NULL); + pThis->pLstnInfo = pLstnInfo; + RETiRet; +} + + +static rsRetVal +SetUsrP(strms_sess_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + + +/* Closes a STRM session + * No attention is paid to the return code + * of close, so potential-double closes are not detected. + */ +static rsRetVal +Close(strms_sess_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strms_sess); + netstrm.Destruct(&pThis->pStrm); + free(pThis->fromHost); + pThis->fromHost = NULL; /* not really needed, but... */ + free(pThis->fromHostIP); + pThis->fromHostIP = NULL; /* not really needed, but... */ + + RETiRet; +} + + + +/* Processes the data received via a STRM session. If there + * is no other way to handle it, data is discarded. + * Input parameter data is the data received, iLen is its + * len as returned from recv(). iLen must be 1 or more (that + * is errors must be handled by caller!). iSTRMSess must be + * the index of the STRM session that received the data. + * rgerhards 2005-07-04 + * And another change while generalizing. We now return either + * RS_RET_OK, which means the session should be kept open + * or anything else, which means it must be closed. + * rgerhards, 2008-03-01 + */ +static rsRetVal +DataRcvd(strms_sess_t *pThis, char *pData, size_t iLen) +{ + DEFiRet; + char *pEnd; + + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pData != NULL); + assert(iLen > 0); + + /* We now copy the message to the session buffer. */ + pEnd = pData + iLen; /* this is one off, which is intensional */ + + while(pData < pEnd) { + CHKiRet(pThis->pSrv->OnCharRcvd(pThis, (uchar)*pData++)); + } + +finalize_it: + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strms_sess) +CODESTARTobjQueryInterface(strms_sess) + if(pIf->ifVersion != strms_sessCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strms_sessDebugPrint; + pIf->Construct = strms_sessConstruct; + pIf->ConstructFinalize = strms_sessConstructFinalize; + pIf->Destruct = strms_sessDestruct; + + pIf->Close = Close; + pIf->DataRcvd = DataRcvd; + + pIf->SetUsrP = SetUsrP; + pIf->SetStrmsrv = SetStrmsrv; + pIf->SetLstnInfo = SetLstnInfo; + pIf->SetHost = SetHost; + pIf->SetHostIP = SetHostIP; + pIf->SetStrm = SetStrm; +finalize_it: +ENDobjQueryInterface(strms_sess) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strms_sess, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strms_sess) + /* release objects we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(datetime, CORE_COMPONENT); +ENDObjClassExit(strms_sess) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strms_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + CHKiRet(objUse(glbl, CORE_COMPONENT)); + iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ + objRelease(glbl, CORE_COMPONENT); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strms_sessDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strms_sessConstructFinalize); +ENDObjClassInit(strms_sess) + +/* vim:set ai: + */ diff --git a/runtime/strms_sess.h b/runtime/strms_sess.h new file mode 100644 index 00000000..c8cb5349 --- /dev/null +++ b/runtime/strms_sess.h @@ -0,0 +1,74 @@ +/* Definitions for strms_sess class. This implements a session of the + * generic stream server. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMS_SESS_H +#define INCLUDED_STRMS_SESS_H + +#include "obj.h" + +/* a forward-definition, we are somewhat cyclic */ +struct strmsrv_s; + +/* the strms_sess object */ +struct strms_sess_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + strmsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ + strmLstnPortList_t *pLstnInfo; /* pointer back to listener info */ + netstrm_t *pStrm; +// uchar *pMsg; /* message (fragment) received */ + uchar *fromHost; + uchar *fromHostIP; + void *pUsr; /* a user-pointer */ +}; + + +/* interfaces */ +BEGINinterface(strms_sess) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strms_sess); + rsRetVal (*Construct)(strms_sess_t **ppThis); + rsRetVal (*ConstructFinalize)(strms_sess_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strms_sess_t **ppThis); + rsRetVal (*Close)(strms_sess_t *pThis); + rsRetVal (*DataRcvd)(strms_sess_t *pThis, char *pData, size_t iLen); + /* set methods */ + rsRetVal (*SetStrmsrv)(strms_sess_t *pThis, struct strmsrv_s *pSrv); + rsRetVal (*SetLstnInfo)(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo); + rsRetVal (*SetUsrP)(strms_sess_t*, void*); + rsRetVal (*SetHost)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetHostIP)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetStrm)(strms_sess_t *pThis, netstrm_t*); + rsRetVal (*SetOnMsgReceive)(strms_sess_t *pThis, rsRetVal (*OnMsgReceive)(strms_sess_t*, uchar*, int)); +ENDinterface(strms_sess) +#define strms_sessCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface changes + * to version v2, rgerhards, 2009-05-22 + * - Data structures changed + * - SetLstnInfo entry point added + */ + + +/* prototypes */ +PROTOTYPEObj(strms_sess); + + +#endif /* #ifndef INCLUDED_STRMS_SESS_H */ diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c new file mode 100644 index 00000000..6c803ee2 --- /dev/null +++ b/runtime/strmsrv.c @@ -0,0 +1,952 @@ +/* strmsrv.c + * + * This builds a basic stream server. It handles connection creation but + * not any protocol. Instead, it calls a "data received" entry point of the + * caller with any data received, in which case the caller must react accordingly. + * This module works together with the netstream drivers. + * + * There are actually two classes within the stream server code: one is + * the strmsrv itself, the other one is its sessions. This is a helper + * class to strmsrv. + * + * File begun on 2009-06-01 by RGerhards based on strmsrv.c. Note that strmsrv is + * placed under LGPL, which is possible because I carefully evaluated and + * eliminated all those parts of strmsrv which were not written by me. + * + * TODO: I would consider it useful to migrate tcpsrv.c/tcps_sess.c to this stream + * class here. The requires a little bit redesign, but should not be too hard. The + * core idea, already begun here, is that we still support lots of callbacks, but + * provide "canned" implementations for standard cases. That way, most upper-layer + * modules can be kept rather simple and without any extra overhead. Note that + * to support this, tcps_sess.c would need to extract the message reception state + * machine to a separate module which then is called via the DoCharRcvd() interface + * of this class here. -- rgerhards, 2009-06-01 + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HAVE_FCNTL_H +#include +#endif +#include "rsyslog.h" +#include "dirty.h" +#include "cfsysline.h" +#include "module-template.h" +#include "net.h" +#include "srUtils.h" +#include "conf.h" +#include "strmsrv.h" +#include "obj.h" +#include "glbl.h" +#include "netstrms.h" +#include "netstrm.h" +#include "nssel.h" +#include "errmsg.h" +#include "unicode-helper.h" + +MODULE_TYPE_LIB + +/* defines */ +#define STRMSESS_MAX_DEFAULT 200 /* default for nbr of strm sessions if no number is given */ +#define STRMLSTN_MAX_DEFAULT 20 /* default for nbr of listeners */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(conf) +DEFobjCurrIf(glbl) +DEFobjCurrIf(strms_sess) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(net) +DEFobjCurrIf(netstrms) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(nssel) + +/* forward definitions */ +static rsRetVal create_strm_socket(strmsrv_t *pThis); + +/* standard callbacks, if the caller did not provide us with them (this helps keep us + * flexible while at the same time permits very simple upper-layer modules) + */ +/* this shall go into a specific ACL module! */ +static int +isPermittedHost(struct sockaddr __attribute__((unused)) *addr, char __attribute__((unused)) *fromHostFQDN, + void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) +{ + return 1; +} + + +static rsRetVal +doOpenLstnSocks(strmsrv_t *pSrv) +{ + ISOBJ_TYPE_assert(pSrv, strmsrv); + return create_strm_socket(pSrv); +} + + +static rsRetVal +doRcvData(strms_sess_t *pSess, char *buf, size_t lenBuf, ssize_t *piLenRcvd) +{ + DEFiRet; + assert(pSess != NULL); + assert(piLenRcvd != NULL); + + *piLenRcvd = lenBuf; + CHKiRet(netstrm.Rcv(pSess->pStrm, (uchar*) buf, piLenRcvd)); +finalize_it: + RETiRet; +} + +static rsRetVal +onRegularClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + /* process any incomplete frames left over */ + //strms_sess.PrepareClose(pSess); + /* Session closed */ + strms_sess.Close(pSess); + RETiRet; +} + + +static rsRetVal +onErrClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + strms_sess.Close(pSess); + RETiRet; +} + +/* ------------------------------ end callbacks ------------------------------ */ + +/* add new listener port to listener port list + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +addNewLstnPort(strmsrv_t *pThis, uchar *pszPort) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* create entry */ + CHKmalloc(pEntry = malloc(sizeof(strmLstnPortList_t))); + pEntry->pszPort = pszPort; + pEntry->pSrv = pThis; + CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + + /* and add to list */ + pEntry->pNext = pThis->pLstnPorts; + pThis->pLstnPorts = pEntry; + +finalize_it: + RETiRet; +} + + +/* configure STRM listener settings. + * Note: pszPort is handed over to us - the caller MUST NOT free it! + * rgerhards, 2008-03-20 + */ +static rsRetVal +configureSTRMListen(strmsrv_t *pThis, uchar *pszPort) +{ + int i; + uchar *pPort = pszPort; + DEFiRet; + + assert(pszPort != NULL); + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* extract port */ + i = 0; + while(isdigit((int) *pPort)) { + i = i * 10 + *pPort++ - '0'; + } + + if(i >= 0 && i <= 65535) { + CHKiRet(addNewLstnPort(pThis, pszPort)); + } else { + errmsg.LogError(0, NO_ERRCODE, "Invalid STRM listen port %s - ignored.\n", pszPort); + } + +finalize_it: + RETiRet; +} + + +/* Initialize the session table + * returns 0 if OK, somewhat else otherwise + */ +static rsRetVal +STRMSessTblInit(strmsrv_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions == NULL); + + dbgprintf("Allocating buffer for %d STRM sessions.\n", pThis->iSessMax); + if((pThis->pSessions = (strms_sess_t **) calloc(pThis->iSessMax, sizeof(strms_sess_t *))) == NULL) { + dbgprintf("Error: STRMSessInit() could not alloc memory for STRM session table.\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + +finalize_it: + RETiRet; +} + + +/* find a free spot in the session table. If the table + * is full, -1 is returned, else the index of the free + * entry (0 or higher). + */ +static int +STRMSessTblFindFreeSpot(strmsrv_t *pThis) +{ + register int i; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + for(i = 0 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] == NULL) + break; + } + + return((i < pThis->iSessMax) ? i : -1); +} + + +/* Get the next session index. Free session tables entries are + * skipped. This function is provided the index of the last + * session entry, or -1 if no previous entry was obtained. It + * returns the index of the next session or -1, if there is no + * further entry in the table. Please note that the initial call + * might as well return -1, if there is no session at all in the + * session table. + */ +static int +STRMSessGetNxtSess(strmsrv_t *pThis, int iCurr) +{ + register int i; + + BEGINfunc + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions != NULL); + for(i = iCurr + 1 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] != NULL) + break; + } + + ENDfunc + return((i < pThis->iSessMax) ? i : -1); +} + + +/* De-Initialize STRM listner sockets. + * This function deinitializes everything, including freeing the + * session table. No STRM listen receive operations are permitted + * unless the subsystem is reinitialized. + * rgerhards, 2007-06-21 + */ +static void deinit_strm_listener(strmsrv_t *pThis) +{ + int i; + strmLstnPortList_t *pEntry; + strmLstnPortList_t *pDel; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + if(pThis->pSessions != NULL) { + /* close all STRM connections! */ + i = STRMSessGetNxtSess(pThis, -1); + while(i != -1) { + strms_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = STRMSessGetNxtSess(pThis, i); + } + + /* we are done with the session table - so get rid of it... */ + free(pThis->pSessions); + pThis->pSessions = NULL; /* just to make sure... */ + } + + /* free list of strm listen ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + free(pEntry->pszPort); + free(pEntry->pszInputName); + pDel = pEntry; + pEntry = pEntry->pNext; + free(pDel); + } + + /* finally close our listen streams */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + netstrm.Destruct(pThis->ppLstn + i); + } +} + + +/* add a listen socket to our listen socket array. This is a callback + * invoked from the netstrm class. -- rgerhards, 2008-04-23 + */ +static rsRetVal +addStrmLstn(void *pUsr, netstrm_t *pLstn) +{ + strmLstnPortList_t *pPortList = (strmLstnPortList_t *) pUsr; + strmsrv_t *pThis = pPortList->pSrv; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + ISOBJ_TYPE_assert(pLstn, netstrm); + + if(pThis->iLstnMax >= STRMLSTN_MAX_DEFAULT) + ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); + + pThis->ppLstn[pThis->iLstnMax] = pLstn; + pThis->ppLstnPort[pThis->iLstnMax] = pPortList; + ++pThis->iLstnMax; + +finalize_it: + RETiRet; +} + + +/* Initialize STRM listener socket for a single port + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +initSTRMListener(strmsrv_t *pThis, strmLstnPortList_t *pPortEntry) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pPortEntry != NULL); + + /* TODO: add capability to specify local listen address! */ + CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pPortEntry, addStrmLstn, pPortEntry->pszPort, NULL, pThis->iSessMax)); + +finalize_it: + RETiRet; +} + + +/* Initialize STRM sockets (for listener) and listens on them */ +static rsRetVal +create_strm_socket(strmsrv_t *pThis) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* init all configured ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + CHKiRet(initSTRMListener(pThis, pEntry)); + pEntry = pEntry->pNext; + } + + /* OK, we had success. Now it is also time to + * initialize our connections + */ + if(STRMSessTblInit(pThis) != 0) { + /* OK, we are in some trouble - we could not initialize the + * session table, so we can not continue. We need to free all + * we have assigned so far, because we can not really use it... + */ + errmsg.LogError(0, RS_RET_ERR, "Could not initialize STRM session table, suspending STRM message reception."); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Accept new STRM connection; make entry in session table. If there + * is no more space left in the connection table, the new STRM + * connection is immediately dropped. + * ppSess has a pointer to the newly created session, if it succeeds. + * If it does not succeed, no session is created and ppSess is + * undefined. If the user has provided an OnSessAccept Callback, + * this one is executed immediately after creation of the + * session object, so that it can do its own initialization. + * rgerhards, 2008-03-02 + */ +static rsRetVal +SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSess, netstrm_t *pStrm) +{ + DEFiRet; + strms_sess_t *pSess = NULL; + netstrm_t *pNewStrm = NULL; + int iSess = -1; + struct sockaddr_storage *addr; + uchar *fromHostFQDN = NULL; + uchar *fromHostIP = NULL; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pLstnInfo != NULL); + + CHKiRet(netstrm.AcceptConnReq(pStrm, &pNewStrm)); + + /* Add to session list */ + iSess = STRMSessTblFindFreeSpot(pThis); + if(iSess == -1) { + errno = 0; + errmsg.LogError(0, RS_RET_MAX_SESS_REACHED, "too many strm sessions - dropping incoming request"); + ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); + } + + /* we found a free spot and can construct our session object */ + CHKiRet(strms_sess.Construct(&pSess)); + CHKiRet(strms_sess.SetStrmsrv(pSess, pThis)); + CHKiRet(strms_sess.SetLstnInfo(pSess, pLstnInfo)); + + /* get the host name */ + CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); + CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); + CHKiRet(netstrm.GetRemAddr(pNewStrm, &addr)); + /* TODO: check if we need to strip the domain name here -- rgerhards, 2008-04-24 */ + + /* Here we check if a host is permitted to send us messages. If it isn't, we do not further + * process the message but log a warning (if we are configured to do this). + * rgerhards, 2005-09-26 + */ + if(pThis->pIsPermittedHost != NULL + && !pThis->pIsPermittedHost((struct sockaddr*) addr, (char*) fromHostFQDN, pThis->pUsr, pSess->pUsr)) { + dbgprintf("%s is not an allowed sender\n", fromHostFQDN); + if(glbl.GetOption_DisallowWarning()) { + errno = 0; + errmsg.LogError(0, RS_RET_HOST_NOT_PERMITTED, "STRM message from disallowed sender %s discarded", fromHostFQDN); + } + ABORT_FINALIZE(RS_RET_HOST_NOT_PERMITTED); + } + + /* OK, we have an allowed sender, so let's continue, what + * means we can finally fill in the session object. + */ + CHKiRet(strms_sess.SetHost(pSess, fromHostFQDN)); + fromHostFQDN = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetHostIP(pSess, fromHostIP)); + fromHostIP = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetStrm(pSess, pNewStrm)); + pNewStrm = NULL; /* prevent it from being freed in error handler, now done in strms_sess! */ + CHKiRet(strms_sess.ConstructFinalize(pSess)); + + /* check if we need to call our callback */ + if(pThis->pOnSessAccept != NULL) { + CHKiRet(pThis->pOnSessAccept(pThis, pSess)); + } + + *ppSess = pSess; + pThis->pSessions[iSess] = pSess; + pSess = NULL; /* this is now also handed over */ + +finalize_it: + if(iRet != RS_RET_OK) { + if(pSess != NULL) + strms_sess.Destruct(&pSess); + if(pNewStrm != NULL) + netstrm.Destruct(&pNewStrm); + free(fromHostFQDN); + free(fromHostIP); + } + + RETiRet; +} + + +static void +RunCancelCleanup(void *arg) +{ + nssel_t **ppSel = (nssel_t**) arg; + + if(*ppSel != NULL) + nssel.Destruct(ppSel); +} + + +/* This function is called to gather input. */ +#pragma GCC diagnostic ignored "-Wempty-body" +static rsRetVal +Run(strmsrv_t *pThis) +{ + DEFiRet; + int nfds; + int i; + int iSTRMSess; + int bIsReady; + strms_sess_t *pNewSess; + nssel_t *pSel; + ssize_t iRcvd; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerharsd, 20080-04-24 + */ + pthread_cleanup_push(RunCancelCleanup, (void*) &pSel); + while(1) { + CHKiRet(nssel.Construct(&pSel)); + // TODO: set driver + CHKiRet(nssel.ConstructFinalize(pSel)); + + /* Add the STRM listen sockets to the list of read descriptors. */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.Add(pSel, pThis->ppLstn[i], NSDSEL_RD)); + } + + /* do the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(iSTRMSess != -1) { + /* TODO: access to pNsd is NOT really CLEAN, use method... */ + CHKiRet(nssel.Add(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD)); + /* now get next... */ + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + + /* wait for io to become ready */ + CHKiRet(nssel.Wait(pSel, &nfds)); + + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + --nfds; /* indicate we have processed one */ + } + } + + /* now check the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(nfds && iSTRMSess != -1) { + CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + dbgprintf("netstream %p with new data\n", pThis->pSessions[iSTRMSess]->pStrm); + + /* Receive message */ + iRet = pThis->pRcvData(pThis->pSessions[iSTRMSess], buf, sizeof(buf), &iRcvd); + switch(iRet) { + case RS_RET_CLOSED: + pThis->pOnRegularClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + case RS_RET_RETRY: + /* we simply ignore retry - this is not an error, but we also have not received anything */ + break; + case RS_RET_OK: + /* valid data received, process it! */ + if(strms_sess.DataRcvd(pThis->pSessions[iSTRMSess], buf, iRcvd) != RS_RET_OK) { + /* in this case, something went awfully wrong. + * We are instructed to terminate the session. + */ + errmsg.LogError(0, NO_ERRCODE, "Tearing down STRM Session %d - see " + "previous messages for reason(s)\n", iSTRMSess); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + } + break; + default: + errno = 0; + errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", + pThis->pSessions[iSTRMSess]->pStrm); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + } + --nfds; /* indicate we have processed one */ + } + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + CHKiRet(nssel.Destruct(&pSel)); +finalize_it: /* this is a very special case - this time only we do not exit the function, + * because that would not help us either. So we simply retry it. Let's see + * if that actually is a better idea. Exiting the loop wasn't we always + * crashed, which made sense (the rest of the engine was not prepared for + * that) -- rgerhards, 2008-05-19 + */ + /*EMPTY*/; + } + + /* note that this point is usually not reached */ + pthread_cleanup_pop(0); /* remove cleanup handler */ + + RETiRet; +} +#pragma GCC diagnostic warning "-Wempty-body" + + +/* Standard-Constructor */ +BEGINobjConstruct(strmsrv) /* be sure to specify the object type also in END macro! */ + pThis->iSessMax = STRMSESS_MAX_DEFAULT; /* TODO: useful default ;) */ + /* set default callbacks (used if caller does not overwrite them) */ + pThis->pIsPermittedHost = isPermittedHost; + pThis->OpenLstnSocks = doOpenLstnSocks; + pThis->pRcvData = doRcvData; + pThis->pOnRegularClose = onRegularClose; + pThis->pOnErrClose = onErrClose; + /* session specific callbacks */ + //pThis->OnSessConstructFinalize = + //pThis->pOnSessDestruct = +ENDobjConstruct(strmsrv) + + +/* ConstructionFinalizer */ +static rsRetVal +strmsrvConstructFinalize(strmsrv_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* prepare network stream subsystem */ + CHKiRet(netstrms.Construct(&pThis->pNS)); + CHKiRet(netstrms.SetDrvrMode(pThis->pNS, pThis->iDrvrMode)); + if(pThis->pszDrvrAuthMode != NULL) + CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); + if(pThis->pPermPeers != NULL) + CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); + // TODO: set driver! + CHKiRet(netstrms.ConstructFinalize(pThis->pNS)); + + /* set up listeners */ + CHKmalloc(pThis->ppLstn = calloc(STRMLSTN_MAX_DEFAULT, sizeof(netstrm_t*))); + CHKmalloc(pThis->ppLstnPort = calloc(STRMLSTN_MAX_DEFAULT, sizeof(strmLstnPortList_t*))); + iRet = pThis->OpenLstnSocks(pThis); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + } + RETiRet; +} + + +/* destructor for the strmsrv object */ +BEGINobjDestruct(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strmsrv) + if(pThis->OnDestruct != NULL) + pThis->OnDestruct(pThis->pUsr); + + deinit_strm_listener(pThis); + + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + free(pThis->pszDrvrAuthMode); + free(pThis->ppLstn); + free(pThis->ppLstnPort); + free(pThis->pszInputName); +ENDobjDestruct(strmsrv) + + +/* debugprint for the strmsrv object */ +BEGINobjDebugPrint(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strmsrv) +ENDobjDebugPrint(strmsrv) + +/* set functions */ +static rsRetVal +SetCBIsPermittedHost(strmsrv_t *pThis, int (*pCB)(struct sockaddr *addr, char *fromHostFQDN, void*, void*)) +{ + DEFiRet; + pThis->pIsPermittedHost = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessAccept(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*, strms_sess_t*)) +{ + DEFiRet; + pThis->pOnSessAccept = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessConstructFinalize(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnSessConstructFinalize = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->pOnSessDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnRegularClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnRegularClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnErrClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnErrClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOpenLstnSocks(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*)) +{ + DEFiRet; + pThis->OpenLstnSocks = pCB; + RETiRet; +} + +static rsRetVal +SetUsrP(strmsrv_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + +static rsRetVal +SetOnCharRcvd(strmsrv_t *pThis, rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar)) +{ + DEFiRet; + assert(OnCharRcvd != NULL); + pThis->OnCharRcvd = OnCharRcvd; + RETiRet; +} + +/* Set the input name to use -- rgerhards, 2008-12-10 */ +static rsRetVal +SetInputName(strmsrv_t *pThis, uchar *name) +{ + uchar *pszName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + if(name == NULL) + pszName = NULL; + else + CHKmalloc(pszName = ustrdup(name)); + free(pThis->pszInputName); + pThis->pszInputName = pszName; +finalize_it: + RETiRet; +} + + +/* here follows a number of methods that shuffle authentication settings down + * to the drivers. Drivers not supporting these settings may return an error + * state. + * -------------------------------------------------------------------------- */ + +/* set the driver mode -- rgerhards, 2008-04-30 */ +static rsRetVal +SetDrvrMode(strmsrv_t *pThis, int iMode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iDrvrMode = iMode; + RETiRet; +} + + +/* set the driver authentication mode -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrAuthMode(strmsrv_t *pThis, uchar *mode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + CHKmalloc(pThis->pszDrvrAuthMode = ustrdup(mode)); +finalize_it: + RETiRet; +} + + +/* set the driver's permitted peers -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrPermPeers(strmsrv_t *pThis, permittedPeers_t *pPermPeers) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->pPermPeers = pPermPeers; + RETiRet; +} + + +/* End of methods to shuffle autentication settings to the driver.; + + * -------------------------------------------------------------------------- */ + + +/* set max number of sessions + * this must be called before ConstructFinalize, or it will have no effect! + * rgerhards, 2009-04-09 + */ +static rsRetVal +SetSessMax(strmsrv_t *pThis, int iMax) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iSessMax = iMax; + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strmsrv) +CODESTARTobjQueryInterface(strmsrv) + if(pIf->ifVersion != strmsrvCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strmsrvDebugPrint; + pIf->Construct = strmsrvConstruct; + pIf->ConstructFinalize = strmsrvConstructFinalize; + pIf->Destruct = strmsrvDestruct; + + pIf->configureSTRMListen = configureSTRMListen; + pIf->create_strm_socket = create_strm_socket; + pIf->Run = Run; + + pIf->SetUsrP = SetUsrP; + pIf->SetInputName = SetInputName; + pIf->SetSessMax = SetSessMax; + pIf->SetDrvrMode = SetDrvrMode; + pIf->SetDrvrAuthMode = SetDrvrAuthMode; + pIf->SetDrvrPermPeers = SetDrvrPermPeers; + pIf->SetCBIsPermittedHost = SetCBIsPermittedHost; + pIf->SetCBOpenLstnSocks = SetCBOpenLstnSocks; + pIf->SetCBOnSessAccept = SetCBOnSessAccept; + pIf->SetCBOnSessConstructFinalize = SetCBOnSessConstructFinalize; + pIf->SetCBOnSessDestruct = SetCBOnSessDestruct; + pIf->SetCBOnDestruct = SetCBOnDestruct; + pIf->SetCBOnRegularClose = SetCBOnRegularClose; + pIf->SetCBOnErrClose = SetCBOnErrClose; + pIf->SetOnCharRcvd = SetOnCharRcvd; + +finalize_it: +ENDobjQueryInterface(strmsrv) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strmsrv, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strmsrv) + /* release objects we no longer need */ + objRelease(strms_sess, DONT_LOAD_LIB); + objRelease(conf, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrms, DONT_LOAD_LIB); + objRelease(nssel, DONT_LOAD_LIB); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(net, LM_NET_FILENAME); +ENDObjClassExit(strmsrv) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strmsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(net, LM_NET_FILENAME)); + CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); + CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(strms_sess, DONT_LOAD_LIB)); + CHKiRet(objUse(conf, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strmsrvDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmsrvConstructFinalize); +ENDObjClassInit(strmsrv) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit + /* de-init in reverse order! */ + strmsrvClassExit(); + strms_sessClassExit(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + /* Initialize all classes that are in our module - this includes ourselfs */ + CHKiRet(strms_sessClassInit(pModInfo)); + CHKiRet(strmsrvClassInit(pModInfo)); /* must be done after strms_sess, as we use it */ +ENDmodInit + +/* vim:set ai: + */ diff --git a/runtime/strmsrv.h b/runtime/strmsrv.h new file mode 100644 index 00000000..e63d3a47 --- /dev/null +++ b/runtime/strmsrv.h @@ -0,0 +1,110 @@ +/* Definitions for strmsrv class. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMSRV_H +#define INCLUDED_STRMSRV_H + +#include "obj.h" +#include "strms_sess.h" + +/* list of strm listen ports */ +struct strmLstnPortList_s { + uchar *pszPort; /**< the ports the listener shall listen on */ + uchar *pszInputName; /**< value to be used as input name */ + strmsrv_t *pSrv; /**< pointer to higher-level server instance */ + strmLstnPortList_t *pNext; /**< next port or NULL */ +}; + + +/* the strmsrv object */ +struct strmsrv_s { + BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ + netstrms_t *pNS; /**< pointer to network stream subsystem */ + int iDrvrMode; /**< mode of the stream driver to use */ + uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ + uchar *pszInputName; /**< value to be used as input name */ + permittedPeers_t *pPermPeers;/**< driver's permitted peers */ + int iLstnMax; /**< max nbr of listeners currently supported */ + netstrm_t **ppLstn; /**< our netstream listners */ + strmLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ + int iSessMax; /**< max number of sessions supported */ + strmLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ + int addtlFrameDelim; /**< additional frame delimiter for plain STRM syslog framing (e.g. to handle NetScreen) */ + strms_sess_t **pSessions;/**< array of all of our sessions */ + void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ + /* callbacks */ + int (*pIsPermittedHost)(struct sockaddr *addr, char *fromHostFQDN, void*pUsrSrv, void*pUsrSess); + rsRetVal (*pRcvData)(strms_sess_t*, char*, size_t, ssize_t *); + rsRetVal (*OpenLstnSocks)(struct strmsrv_s*); + rsRetVal (*pOnListenDeinit)(void*); + rsRetVal (*OnDestruct)(void*); + rsRetVal (*pOnRegularClose)(strms_sess_t *pSess); + rsRetVal (*pOnErrClose)(strms_sess_t *pSess); + /* session specific callbacks */ + rsRetVal (*pOnSessAccept)(strmsrv_t *, strms_sess_t*); + rsRetVal (*OnSessConstructFinalize)(void*); + rsRetVal (*pOnSessDestruct)(void*); + rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar); +}; + + +/* interfaces */ +BEGINinterface(strmsrv) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strmsrv); + rsRetVal (*Construct)(strmsrv_t **ppThis); + rsRetVal (*ConstructFinalize)(strmsrv_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strmsrv_t **ppThis); + rsRetVal (*configureSTRMListen)(strmsrv_t*, uchar *pszPort); + //rsRetVal (*SessAccept)(strmsrv_t *pThis, strmLstnPortList_t*, strms_sess_t **ppSess, netstrm_t *pStrm); + rsRetVal (*create_strm_socket)(strmsrv_t *pThis); + rsRetVal (*Run)(strmsrv_t *pThis); + /* set methods */ + rsRetVal (*SetAddtlFrameDelim)(strmsrv_t*, int); + rsRetVal (*SetInputName)(strmsrv_t*, uchar*); + rsRetVal (*SetUsrP)(strmsrv_t*, void*); + rsRetVal (*SetCBIsPermittedHost)(strmsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); + rsRetVal (*SetCBOpenLstnSocks)(strmsrv_t *, rsRetVal (*)(strmsrv_t*)); + rsRetVal (*SetCBOnDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnRegularClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetCBOnErrClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetDrvrMode)(strmsrv_t *pThis, int iMode); + rsRetVal (*SetDrvrAuthMode)(strmsrv_t *pThis, uchar *pszMode); + rsRetVal (*SetDrvrPermPeers)(strmsrv_t *pThis, permittedPeers_t*); + /* session specifics */ + rsRetVal (*SetCBOnSessAccept)(strmsrv_t*, rsRetVal (*) (strmsrv_t*, strms_sess_t*)); + rsRetVal (*SetCBOnSessDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnSessConstructFinalize)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetSessMax)(strmsrv_t *pThis, int iMaxSess); + rsRetVal (*SetOnCharRcvd)(strmsrv_t *pThis, rsRetVal (*OnMsgCharRcvd)(strms_sess_t*, uchar)); +ENDinterface(strmsrv) +#define strmsrvCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* change for v?: + */ + + +/* prototypes */ +PROTOTYPEObj(strmsrv); + +/* the name of our library binary */ +#define LM_STRMSRV_FILENAME "lmstrmsrv" + +#endif /* #ifndef INCLUDED_STRMSRV_H */ diff --git a/tcpsrv.c b/tcpsrv.c index 02eee88e..3516b2e3 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -1,8 +1,11 @@ /* tcpsrv.c * - * Common code for plain TCP based servers. This is currently being + * Common code for plain TCP syslog based servers. This is currently being * utilized by imtcp and imgssapi. * + * NOTE: this is *not* a generic TCP server, but one for syslog servers. For + * generic stream servers, please use ./runtime/strmsrv.c! + * * There are actually two classes within the tcpserver code: one is * the tcpsrv itself, the other one is its sessions. This is a helper * class to tcpsrv. -- cgit From 05ba5fc29fb13857b6fbb7c8f3c22489da3e73cf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 12:22:00 +0200 Subject: slightly extended strms_sess interface --- runtime/strms_sess.c | 10 +++++++++- runtime/strms_sess.h | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c index 20920430..0aeebe03 100644 --- a/runtime/strms_sess.c +++ b/runtime/strms_sess.c @@ -64,7 +64,7 @@ ENDobjConstruct(strms_sess) /* ConstructionFinalizer */ static rsRetVal -strms_sessConstructFinalize(strms_sess_t __attribute__((unused)) *pThis) +strms_sessConstructFinalize(strms_sess_t *pThis) { DEFiRet; ISOBJ_TYPE_assert(pThis, strms_sess); @@ -170,6 +170,13 @@ SetUsrP(strms_sess_t *pThis, void *pUsr) } +static void * +GetUsrP(strms_sess_t *pThis) +{ + return pThis->pUsr; +} + + /* Closes a STRM session * No attention is paid to the return code * of close, so potential-double closes are not detected. @@ -248,6 +255,7 @@ CODESTARTobjQueryInterface(strms_sess) pIf->DataRcvd = DataRcvd; pIf->SetUsrP = SetUsrP; + pIf->GetUsrP = GetUsrP; pIf->SetStrmsrv = SetStrmsrv; pIf->SetLstnInfo = SetLstnInfo; pIf->SetHost = SetHost; diff --git a/runtime/strms_sess.h b/runtime/strms_sess.h index c8cb5349..6483c0c3 100644 --- a/runtime/strms_sess.h +++ b/runtime/strms_sess.h @@ -54,6 +54,7 @@ BEGINinterface(strms_sess) /* name must also be changed in ENDinterface macro! * rsRetVal (*SetStrmsrv)(strms_sess_t *pThis, struct strmsrv_s *pSrv); rsRetVal (*SetLstnInfo)(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo); rsRetVal (*SetUsrP)(strms_sess_t*, void*); + void* (*GetUsrP)(strms_sess_t*); rsRetVal (*SetHost)(strms_sess_t *pThis, uchar*); rsRetVal (*SetHostIP)(strms_sess_t *pThis, uchar*); rsRetVal (*SetStrm)(strms_sess_t *pThis, netstrm_t*); -- cgit From b61e5fffc42c98b49a18a95c297653fb0ba06b72 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 13:11:18 +0200 Subject: strmsrv now supports KEEPALIVE socket option --- runtime/netstrm.c | 14 ++++++++++++++ runtime/netstrm.h | 8 ++++++-- runtime/nsd.h | 8 ++++++-- runtime/nsd_gtls.c | 11 +++++++++++ runtime/nsd_ptcp.c | 29 +++++++++++++++++++++++++++++ runtime/strmsrv.c | 14 ++++++++++++++ runtime/strmsrv.h | 2 ++ 7 files changed, 82 insertions(+), 4 deletions(-) diff --git a/runtime/netstrm.c b/runtime/netstrm.c index 05bb25c0..3658006f 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -233,6 +233,19 @@ Send(netstrm_t *pThis, uchar *pBuf, ssize_t *pLenBuf) RETiRet; } +/* Enable Keep-Alive handling for those drivers that support it. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(netstrm_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, netstrm); + iRet = pThis->Drvr.EnableKeepAlive(pThis->pDrvrData); + RETiRet; +} + + /* check connection - slim wrapper for NSD driver function */ static void @@ -337,6 +350,7 @@ CODESTARTobjQueryInterface(netstrm) pIf->SetDrvrPermPeers = SetDrvrPermPeers; pIf->CheckConnection = CheckConnection; pIf->GetSock = GetSock; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(netstrm) diff --git a/runtime/netstrm.h b/runtime/netstrm.h index b00dd223..f6931104 100644 --- a/runtime/netstrm.h +++ b/runtime/netstrm.h @@ -69,9 +69,13 @@ BEGINinterface(netstrm) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v4 */ + rsRetVal (*EnableKeepAlive)(netstrm_t *pThis); ENDinterface(netstrm) -#define netstrmCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ -/* interface version 3 added GetRemAddr() */ +#define netstrmCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +/* interface version 3 added GetRemAddr() + * interface version 4 added EnableKeepAlive() -- rgerhards, 2009-06-02 + * */ /* prototypes */ PROTOTYPEObj(netstrm); diff --git a/runtime/nsd.h b/runtime/nsd.h index f0c9b9b6..8668c934 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -69,9 +69,13 @@ BEGINinterface(nsd) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v5 */ + rsRetVal (*EnableKeepAlive)(nsd_t *pThis); ENDinterface(nsd) -#define nsdCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ -/* interface version 4 added GetRemAddr() */ +#define nsdCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +/* interface version 4 added GetRemAddr() + * interface version 5 added EnableKeepAlive() -- rgerhards, 2009-06-02 + */ /* interface for the select call */ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 5786e191..1a50e2f8 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -1560,6 +1560,16 @@ finalize_it: RETiRet; } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + return nsd_ptcp.EnableKeepAlive(pNsd); +} + + /* open a connection to a remote host (server). With GnuTLS, we always * open a plain tcp socket and then, if in TLS mode, do a handshake on it. @@ -1669,6 +1679,7 @@ CODESTARTobjQueryInterface(nsd_gtls) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->GetRemAddr = GetRemAddr; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_gtls) diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index cc531ca0..54ee0666 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -614,6 +614,34 @@ finalize_it: } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; + int ret; + int optval; + socklen_t optlen; + DEFiRet; + ISOBJ_TYPE_assert(pThis, nsd_ptcp); + + optval = 1; + optlen = sizeof(optval); + ret = setsockopt(pThis->sock, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen); + if(ret < 0) { + dbgprintf("EnableKeepAlive socket call returns error %d\n", ret); + ABORT_FINALIZE(RS_RET_ERR); + } + + dbgprintf("KEEPALIVE enabled for nsd %p\n", pThis); + +finalize_it: + RETiRet; +} + + /* open a connection to a remote host (server). * rgerhards, 2008-03-19 */ @@ -754,6 +782,7 @@ CODESTARTobjQueryInterface(nsd_ptcp) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->CheckConnection = CheckConnection; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_ptcp) diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index 6c803ee2..3dc53a97 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -434,6 +434,10 @@ SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSes ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); } + if(pThis->bUseKeepAlive) { + CHKiRet(netstrm.EnableKeepAlive(pNewStrm)); + } + /* we found a free spot and can construct our session object */ CHKiRet(strms_sess.Construct(&pSess)); CHKiRet(strms_sess.SetStrmsrv(pSess, pThis)); @@ -756,6 +760,15 @@ SetUsrP(strmsrv_t *pThis, void *pUsr) RETiRet; } +static rsRetVal +SetKeepAlive(strmsrv_t *pThis, int iVal) +{ + DEFiRet; + dbgprintf("keep-alive set to %d\n", iVal); + pThis->bUseKeepAlive = iVal; + RETiRet; +} + static rsRetVal SetOnCharRcvd(strmsrv_t *pThis, rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar)) { @@ -864,6 +877,7 @@ CODESTARTobjQueryInterface(strmsrv) pIf->create_strm_socket = create_strm_socket; pIf->Run = Run; + pIf->SetKeepAlive = SetKeepAlive; pIf->SetUsrP = SetUsrP; pIf->SetInputName = SetInputName; pIf->SetSessMax = SetSessMax; diff --git a/runtime/strmsrv.h b/runtime/strmsrv.h index e63d3a47..86e529c3 100644 --- a/runtime/strmsrv.h +++ b/runtime/strmsrv.h @@ -38,6 +38,7 @@ struct strmLstnPortList_s { /* the strmsrv object */ struct strmsrv_s { BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ + int bUseKeepAlive; /**< use socket layer KEEPALIVE handling? */ netstrms_t *pNS; /**< pointer to network stream subsystem */ int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ @@ -80,6 +81,7 @@ BEGINinterface(strmsrv) /* name must also be changed in ENDinterface macro! */ /* set methods */ rsRetVal (*SetAddtlFrameDelim)(strmsrv_t*, int); rsRetVal (*SetInputName)(strmsrv_t*, uchar*); + rsRetVal (*SetKeepAlive)(strmsrv_t*, int); rsRetVal (*SetUsrP)(strmsrv_t*, void*); rsRetVal (*SetCBIsPermittedHost)(strmsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); rsRetVal (*SetCBOpenLstnSocks)(strmsrv_t *, rsRetVal (*)(strmsrv_t*)); -- cgit From 4f742a8e32c43dc9b514ceaf80f4d17e697dfdf6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 07:51:47 +0200 Subject: added config switch --enable-testbench=no to turn off testbench --- ChangeLog | 1 + configure.ac | 15 +++++++++++++++ tests/Makefile.am | 3 +++ 3 files changed, 19 insertions(+) diff --git a/ChangeLog b/ChangeLog index ec971916..f9e3e32b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? considered useful for load balancers. - bugfix: imdiag/imtcp had a race condition - improved testbench (now much better code design and reuse) +- added config switch --enable-testbench=no to turn off testbench --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/configure.ac b/configure.ac index 9fc59f76..70eea038 100644 --- a/configure.ac +++ b/configure.ac @@ -678,6 +678,20 @@ AC_SUBST(LIBLOGGING_CFLAGS) AC_SUBST(LIBLOGGING_LIBS) +# enable/disable the testbench (e.g. because some important parts +# are missing) +AC_ARG_ENABLE(testbench, + [AS_HELP_STRING([--enable-testbench],[file input module enabled @<:@default=yes@:>@])], + [case "${enableval}" in + yes) enable_testbench="yes" ;; + no) enable_testbench="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-testbench) ;; + esac], + [enable_testbench=yes] +) +AM_CONDITIONAL(ENABLE_TESTBENCH, test x$enable_testbench = xyes) + + # settings for the file input module AC_ARG_ENABLE(imfile, [AS_HELP_STRING([--enable-imfile],[file input module enabled @<:@default=no@:>@])], @@ -844,6 +858,7 @@ echo " RELP support enabled: $enable_relp" echo " SNMP support enabled: $enable_snmp" echo echo "---{ debugging support }---" +echo " Testbench enabled: $enable_testbench" echo " Debug mode enabled: $enable_debug" echo " Runtime Instrumentation enabled: $enable_rtinst" echo " Diagnostic tools enabled: $enable_diagtools" diff --git a/tests/Makefile.am b/tests/Makefile.am index e93aba10..402c1d83 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,3 +1,4 @@ +if ENABLE_TESTBENCH TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh queue-persist.sh \ @@ -6,6 +7,8 @@ if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif +endif # if ENABLE_TESTBENCH + TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class test_files = testbench.h runtime-dummy.c -- cgit From 58e707b441aea88cd318762e6968e1db1211f949 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 09:57:45 +0200 Subject: backported some of the v5 testbench this permits us to keep a persistent test environment between v4 and v5, most importantly using the same tools. As far as the actual tests are concerned, some had issues. I had no time to check if that was an issue with the test or an actual issue with the v3/4 engine. Will do that at some later stage. --- runtime/rsyslog.h | 5 ++- tests/DiagTalker.java | 33 +++++++++++++++++-- tests/Makefile.am | 26 +++++++++++---- tests/chkseq.c | 66 ++++++++++++++++++++++++++++--------- tests/diag.sh | 10 +++--- tests/diskqueue.sh | 3 -- tests/ourtail.c | 2 +- tests/rscript.c | 4 +-- tests/tcpflood.c | 13 +++++--- tests/testsuites/queue-persist.conf | 2 -- 10 files changed, 123 insertions(+), 41 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 293e559b..ea303a51 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -273,7 +273,10 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ - RS_RET_NONFATAL_CONFIG_ERR = -2123, /**< non-fatal error during config processing */ + RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ + RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ + RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */ + RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ /* RainerScript error messages (range 1000.. 1999) */ diff --git a/tests/DiagTalker.java b/tests/DiagTalker.java index e33a5867..04e12327 100644 --- a/tests/DiagTalker.java +++ b/tests/DiagTalker.java @@ -1,3 +1,24 @@ +/* A yet very simple tool to talk to imdiag. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ //package com.rsyslog.diag; import java.io.*; import java.net.*; @@ -13,6 +34,7 @@ public class DiagTalker { try { diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(0); /* wait for lenghty operations */ out = new PrintWriter(diagSocket.getOutputStream(), true); in = new BufferedReader(new InputStreamReader( diagSocket.getInputStream())); @@ -29,9 +51,14 @@ public class DiagTalker { new InputStreamReader(System.in)); String userInput; - while ((userInput = stdIn.readLine()) != null) { - out.println(userInput); - System.out.println("imdiag returns: " + in.readLine()); + try { + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + } catch (SocketException e) { + System.err.println("We had a socket exception and consider this to be OK: " + + e.getMessage()); } out.close(); diff --git a/tests/Makefile.am b/tests/Makefile.am index 402c1d83..a95139f2 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,13 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh queue-persist.sh \ - validation-run.sh +TESTS = $(TESTRUNS) cfg.sh \ + validation-run.sh \ + imtcp-multiport.sh \ + diskqueue.sh \ + manytcp.sh \ + queue-persist.sh + if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif @@ -32,25 +37,32 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ err1.rstest \ NoExistFile.cfgtest \ testsuites/parse1.conf \ + testsuites/field1.conf \ testsuites/1.parse1 \ - testsuites/rfc3164.parse1 \ + testsuites/2.parse1 \ + testsuites/3.parse1 \ + testsuites/date1.parse1 \ + testsuites/date2.parse1 \ + testsuites/date3.parse1 \ + testsuites/date4.parse1 \ + testsuites/date5.parse1 \ + testsuites/rfc3164.parse1 \ testsuites/rfc5424-1.parse1 \ testsuites/rfc5424-2.parse1 \ testsuites/rfc5424-3.parse1 \ testsuites/rfc5424-4.parse1 \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ + testsuites/1.field1 \ killrsyslog.sh \ parsertest.sh \ + fieldtest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ imtcp-multiport.sh \ testsuites/imtcp-multiport.conf \ manytcp.sh \ testsuites/manytcp.conf \ - fieldtest.sh \ - testsuites/field1.conf \ - testsuites/1.field1 \ inputname.sh \ testsuites/inputname_imtcp.conf \ testsuites/1.inputname_imtcp_12514 \ @@ -58,6 +70,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ + daqueue-persist.sh \ + daqueue-persist-drvr.sh \ testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ diff --git a/tests/chkseq.c b/tests/chkseq.c index 3203c250..8c5fc61a 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -3,9 +3,10 @@ * be set. * * Params - * argv[1] file to check - * argv[2] start number - * argv[3] end number + * -f MUST be given! + * -s -e + * default for s is 0. -e should be given (else it is also 0) + * -d may be specified, in which case duplicate messages are permitted. * * Part of the testbench for rsyslog. * @@ -31,6 +32,7 @@ #include "config.h" #include #include +#include int main(int argc, char *argv[]) { @@ -38,16 +40,36 @@ int main(int argc, char *argv[]) int val; int i; int ret = 0; - int start, end; + int dupsPermitted = 0; + int start = 0, end = 0; + int opt; + int nDups = 0; + char *file = NULL; - if(argc != 4) { - printf("Invalid call of chkseq\n"); - printf("Usage: chkseq file start end\n"); + while((opt = getopt(argc, argv, "e:f:ds:")) != EOF) { + switch((char)opt) { + case 'f': + file = optarg; + break; + case 'd': + dupsPermitted = 1; + break; + case 'e': + end = atoi(optarg); + break; + case 's': + start = atoi(optarg); + break; + default:printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file -sstart -eend -d\n"); + exit(1); + } + } + + if(file == NULL) { + printf("file must be given!\n"); exit(1); } - - start = atoi(argv[2]); - end = atoi(argv[3]); if(start > end) { printf("start must be less than or equal end!\n"); @@ -55,22 +77,36 @@ int main(int argc, char *argv[]) } /* read file */ - fp = fopen(argv[1], "r"); + fp = fopen(file, "r"); if(fp == NULL) { - perror(argv[1]); + printf("error opening file '%s'\n", file); + perror(file); exit(1); } - for(i = start ; i < end ; ++i) { + for(i = start ; i < end+1 ; ++i) { if(fscanf(fp, "%d\n", &val) != 1) { printf("scanf error in index i=%d\n", i); exit(1); } if(val != i) { - printf("read value %d, but expected value %d\n", val, i); - exit(1); + if(val == i - 1 && dupsPermitted) { + --i; + ++nDups; + } else { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } } } + if(nDups != 0) + printf("info: had %d duplicates (this is no error)\n", nDups); + + if(i - 1 != end) { + printf("only %d records in file, expected %d\n", i - 1, end); + exit(1); + } + exit(ret); } diff --git a/tests/diag.sh b/tests/diag.sh index ec2c1190..1ceca75b 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -5,8 +5,11 @@ # not always able to convey back states to the upper-level test driver # begun 2009-05-27 by rgerhards # This file is part of the rsyslog project, released under GPLv3 +#valgrind="valgrind --log-fd=1" +#valgrind="valgrind --tool=drd --log-fd=1" +#valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUG="debug nostdout printmutexaction" #export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason @@ -22,7 +25,7 @@ case $1 in ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use # returns only after successful startup - ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & $srcdir/diag.sh wait-startup ;; 'wait-startup') # wait for rsyslogd startup @@ -72,9 +75,8 @@ case $1 in 'seq-check') # do the usual sequence check to see if everything was properly received rm -f work sort < rsyslog.out.log > work - ./chkseq work $2 $3 + ./chkseq -fwork -e$2 $3 if [ "$?" -ne "0" ]; then - rm -f work rsyslog.out.log echo "sequence error detected" exit 1 fi diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index bf1a46fd..2fe31db9 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -5,9 +5,6 @@ # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -#set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="tmp" echo testing queue disk-only mode source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue.conf diff --git a/tests/ourtail.c b/tests/ourtail.c index f2751c72..6781b5fe 100644 --- a/tests/ourtail.c +++ b/tests/ourtail.c @@ -28,7 +28,7 @@ */ #include -int main(int argc, char *argv[]) +int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) { int c; diff --git a/tests/rscript.c b/tests/rscript.c index ce81491c..6361aec4 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -104,8 +104,8 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { printf("error: compiled program different from expected result!\n"); - printf("generated vmprg (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); - printf("expected (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } diff --git a/tests/tcpflood.c b/tests/tcpflood.c index c3c9c871..2ca796ca 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -6,6 +6,7 @@ * argv[2] target port * argv[3] number of connections * argv[4] number of messages to send (connection is random) + * argv[5] initial message number (optional) * * Part of the testbench for rsyslog. * @@ -51,6 +52,7 @@ static int targetPort; static int numMsgsToSend; /* number of messages to send */ static int numConnections; /* number of connections to create */ static int *sockArray; /* array of sockets to use */ +static int msgNum = 0; /* initial message number to start with */ /* open a single tcp connection @@ -166,19 +168,20 @@ int sendMessages(void) socknum = i - (numMsgsToSend - numConnections); else socknum = rand() % numConnections; - lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", msgNum); lenSend = send(sockArray[socknum], buf, lenBuf, 0); if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); perror("send test data"); - printf("send() failed at socket %d, index %d\n", socknum, i); + printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum); fflush(stderr); return(1); } if(i % 100 == 0) { printf("\r%5.5d", i); } + ++msgNum; } printf("\r%5.5d messages sent\n", i); @@ -260,9 +263,9 @@ int main(int argc, char *argv[]) setvbuf(stdout, buf, _IONBF, 48); - if(argc != 5) { + if(argc != 5 && argc != 6) { printf("Invalid call of tcpflood\n"); - printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages [initial msgnum]\n"); exit(1); } @@ -270,6 +273,8 @@ int main(int argc, char *argv[]) targetPort = atoi(argv[2]); numConnections = atoi(argv[3]); numMsgsToSend = atoi(argv[4]); + if(argc == 6) + msgNum = atoi(argv[5]); if(openConnections() != 0) { printf("error opening connections\n"); diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf index 81ee1be5..8903042d 100644 --- a/tests/testsuites/queue-persist.conf +++ b/tests/testsuites/queue-persist.conf @@ -9,8 +9,6 @@ $InputTCPServerRun 13514 $ModLoad ../plugins/omtesting/.libs/omtesting -$ErrorMessagesToStderr off - # set spool locations and switch queue to disk-only mode $WorkDirectory test-spool $MainMsgQueueFilename mainq -- cgit From e893a5916473c45d6b90dc2401637fe713133291 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 10:32:12 +0200 Subject: backported macros necessary to create output modules with transaction support this does not mean the v4 engine will support batches and output transactions, but I can now write plugins that will work equally well with v4 AND v5. I consider this useful during the rewrite of omfile. --- runtime/module-template.h | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/runtime/module-template.h b/runtime/module-template.h index 6f7d877c..3e963199 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -39,7 +39,8 @@ #define DEF_OMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ - DEFobjCurrIf(obj) + DEFobjCurrIf(obj) \ + static __attribute__((unused)) int bCoreSupportsBatching; #define DEF_IMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ DEFobjCurrIf(obj) @@ -160,6 +161,37 @@ static rsRetVal isCompatibleWithFeature(syslogFeature __attribute__((unused)) eF RETiRet;\ } + +/* beginTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINbeginTransaction \ +static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTbeginTransaction /* currently empty, but may be extended */ + +#define ENDbeginTransaction \ + RETiRet;\ +} + + +/* endTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINendTransaction \ +static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTendTransaction /* currently empty, but may be extended */ + +#define ENDendTransaction \ + RETiRet;\ +} + + /* doAction() */ #define BEGINdoAction \ @@ -324,6 +356,18 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = tryResume;\ } + +/* the following definition is queryEtryPt block that must be added + * if an output module supports the transactional interface. + * rgerhards, 2009-04-27 + */ +#define CODEqueryEtryPt_TXIF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "beginTransaction")) {\ + *pEtryPoint = beginTransaction;\ + } else if(!strcmp((char*) name, "endTransaction")) {\ + *pEtryPoint = endTransaction;\ + } + /* the following definition is the standard block for queryEtryPt for INPUT * modules. This can be used if no specific handling (e.g. to cover version * differences) is needed. @@ -393,6 +437,32 @@ finalize_it:\ } +/* now come some check functions, which enable a standard way of obtaining feature + * information from the core. feat is the to-be-tested feature and featVar is a + * variable that receives the result (0-not support, 1-supported). + * This must be a macro, so that it is put into the output's code. Otherwise, we + * would need to rely on a library entry point, which is what we intend to avoid ;) + * rgerhards, 2009-04-27 + */ +#define INITChkCoreFeature(featVar, feat) \ +{ \ + rsRetVal MACRO_Ret; \ + rsRetVal (*pQueryCoreFeatureSupport)(int*, unsigned); \ + int bSupportsIt; \ + featVar = 0; \ + MACRO_Ret = pHostQueryEtryPt((uchar*)"queryCoreFeatureSupport", &pQueryCoreFeatureSupport); \ + if(MACRO_Ret == RS_RET_OK) { \ + /* found entry point, so let's see if core supports it */ \ + CHKiRet((*pQueryCoreFeatureSupport)(&bSupportsIt, feat)); \ + if(bSupportsIt) \ + featVar = 1; \ + } else if(MACRO_Ret != RS_RET_ENTRY_POINT_NOT_FOUND) { \ + ABORT_FINALIZE(MACRO_Ret); /* Something else went wrong, what is not acceptable */ \ + } \ +} + + + /* definitions for host API queries */ #define CODEmodInit_QueryRegCFSLineHdlr \ CHKiRet(pHostQueryEtryPt((uchar*)"regCfSysLineHdlr", &omsdRegCFSLineHdlr)); -- cgit From 015d17ca70e81ad998e32cdfeed3cd925fd7dedc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 08:46:45 +0200 Subject: some performance optimizations - saved gettimeofday() calls in imtcp (and increased reception buffer) - somewhat optimized stringbuf.c - some other optimizations --- plugins/imdiag/imdiag.c | 2 +- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- plugins/imudp/imudp.c | 4 ++-- runtime/msg.c | 57 ++++++++++++++++++++----------------------------- runtime/msg.h | 48 ++++++++++++++++++----------------------- runtime/obj-types.h | 4 ++-- runtime/parser.c | 17 +++++++++++++++ runtime/rsyslog.h | 2 ++ runtime/stringbuf.c | 6 ++---- tcps_sess.c | 54 ++++++++++++++++++++++------------------------ tcpsrv.c | 3 ++- tcpsrv.h | 1 + tools/syslogd.c | 4 ++-- 14 files changed, 102 insertions(+), 104 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index c700cab7..51f319ca 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -207,7 +207,7 @@ doInjectMsg(int iNum) CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg)); pMsg->iLenRawMsg = ustrlen(szMsg); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 92fd30c3..e3882ce5 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -95,7 +95,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index ecb6c100..45933436 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -95,7 +95,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsg(pMsg, (char*)msg); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index a486b818..57aec9b6 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -220,7 +220,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); pMsg->iLenRawMsg = lenRcvBuf; - MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp"), sizeof("imudp")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; diff --git a/runtime/msg.c b/runtime/msg.c index dbc3c779..b3c76089 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -104,6 +104,7 @@ static syslogCODE rs_facilitynames[] = /* some forward declarations */ static int getAPPNAMELen(msg_t *pM); +static int getProtocolVersion(msg_t *pM); /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we @@ -364,7 +365,6 @@ CODESTARTobjDestruct(msg) free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); - free(pThis->pszPRI); if(pThis->pCSProgName != NULL) rsCStrDestruct(&pThis->pCSProgName); if(pThis->pCSStrucData != NULL) @@ -670,7 +670,7 @@ void setProtocolVersion(msg_t *pM, int iNewVersion) pM->iProtocolVersion = iNewVersion; } -int getProtocolVersion(msg_t *pM) +static int getProtocolVersion(msg_t *pM) { assert(pM != NULL); return(pM->iProtocolVersion); @@ -689,7 +689,7 @@ int getMSGLen(msg_t *pM) } -char *getRawMsg(msg_t *pM) +static char *getRawMsg(msg_t *pM) { if(pM == NULL) return ""; @@ -724,7 +724,7 @@ char *getMSG(msg_t *pM) /* Get PRI value in text form */ -char *getPRI(msg_t *pM) +static char *getPRI(msg_t *pM) { int pri; BEGINfunc @@ -742,7 +742,7 @@ char *getPRI(msg_t *pM) * report. -- rgerhards, 2008-07-14 */ pri = pM->iFacility * 8 + pM->iSeverity; - if((pM->pszPRI = malloc(5)) == NULL) return ""; + pM->pszPRI = pM->bufPRI; pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); } MsgUnlock(pM); @@ -753,14 +753,14 @@ char *getPRI(msg_t *pM) /* Get PRI value as integer */ -int getPRIi(msg_t *pM) +static int getPRIi(msg_t *pM) { assert(pM != NULL); return (pM->iFacility << 3) + (pM->iSeverity); } -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -838,7 +838,7 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) return "INVALID eFmt OPTION!"; } -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -917,7 +917,7 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) } -char *getSeverity(msg_t *pM) +static inline char *getSeverity(msg_t *pM) { if(pM == NULL) return ""; @@ -934,7 +934,7 @@ char *getSeverity(msg_t *pM) } -char *getSeverityStr(msg_t *pM) +static inline char *getSeverityStr(msg_t *pM) { syslogCODE *c; int val; @@ -964,7 +964,7 @@ char *getSeverityStr(msg_t *pM) return((char*)pM->pszSeverityStr); } -char *getFacility(msg_t *pM) +static inline char *getFacility(msg_t *pM) { if(pM == NULL) return ""; @@ -983,7 +983,7 @@ char *getFacility(msg_t *pM) return((char*)pM->pszFacility); } -char *getFacilityStr(msg_t *pM) +static inline char *getFacilityStr(msg_t *pM) { syslogCODE *c; int val; @@ -1090,7 +1090,7 @@ finalize_it: /* rgerhards, 2005-11-24 */ -int getPROCIDLen(msg_t *pM) +static inline int getPROCIDLen(msg_t *pM) { assert(pM != NULL); MsgLock(pM); @@ -1135,19 +1135,10 @@ finalize_it: RETiRet; } -/* rgerhards, 2005-11-24 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getMSGIDLen(msg_t *pM) -{ - return (pM->pCSMSGID == NULL) ? 1 : rsCStrLen(pM->pCSMSGID); -} -#endif - /* rgerhards, 2005-11-24 */ -char *getMSGID(msg_t *pM) +static inline char *getMSGID(msg_t *pM) { return (pM->pCSMSGID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSMSGID); } @@ -1171,8 +1162,7 @@ void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) void MsgSetTAG(msg_t *pMsg, char* pszTAG) { assert(pMsg != NULL); - if(pMsg->pszTAG != NULL) - free(pMsg->pszTAG); + free(pMsg->pszTAG); pMsg->iLenTAG = strlen(pszTAG); if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); @@ -1229,7 +1219,7 @@ static int getTAGLen(msg_t *pM) #endif -char *getTAG(msg_t *pM) +static inline char *getTAG(msg_t *pM) { char *ret; @@ -1272,7 +1262,7 @@ char *getHOSTNAME(msg_t *pM) } -uchar *getInputName(msg_t *pM) +static uchar *getInputName(msg_t *pM) { if(pM == NULL) return (uchar*) ""; @@ -1339,7 +1329,7 @@ static int getStructuredDataLen(msg_t *pM) /* get the "STRUCTURED-DATA" as sz string * rgerhards, 2005-11-24 */ -char *getStructuredData(msg_t *pM) +static inline char *getStructuredData(msg_t *pM) { return (pM->pCSStrucData == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSStrucData); } @@ -1465,14 +1455,13 @@ static int getAPPNAMELen(msg_t *pM) } /* rgerhards 2008-09-10: set pszInputName in msg object + * rgerhards, 2009-06-16 */ -void MsgSetInputName(msg_t *pMsg, uchar* pszInputName) +void MsgSetInputName(msg_t *pMsg, uchar* pszInputName, size_t lenInputName) { assert(pMsg != NULL); - if(pMsg->pszInputName != NULL) - free(pMsg->pszInputName); - - pMsg->iLenInputName = ustrlen(pszInputName); + free(pMsg->pszInputName); + pMsg->iLenInputName = lenInputName; if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); } @@ -2493,7 +2482,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..1689bbbc 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -120,6 +120,9 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ + /* now follow fixed-size buffers to safe some time otherwise used for allocs */ + uchar bufPRI[5]; + }; @@ -137,45 +140,20 @@ short bDoLock; /* use the mutex? */ /* function prototypes */ PROTOTYPEObjClassInit(msg); -char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -int getProtocolVersion(msg_t *pM); -char *getProtocolVersionString(msg_t *pM); -int getMSGLen(msg_t *pM); -char *getRawMsg(msg_t *pM); -char *getUxTradMsg(msg_t *pM); -char *getMSG(msg_t *pM); -char *getPRI(msg_t *pM); -int getPRIi(msg_t *pM); -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt); -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt); -char *getSeverity(msg_t *pM); -char *getSeverityStr(msg_t *pM); -char *getFacility(msg_t *pM); -char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, uchar*); +void MsgSetInputName(msg_t *pMsg, uchar*, size_t); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); -char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); -int getPROCIDLen(msg_t *pM); -char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, char* pszTAG); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); -char *getTAG(msg_t *pM); -int getHOSTNAMELen(msg_t *pM); -char *getHOSTNAME(msg_t *pM); -uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -char *getStructuredData(msg_t *pM); -int getProgramNameLen(msg_t *pM); -char *getProgramName(msg_t *pM); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); @@ -184,13 +162,29 @@ int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); -char *getMSGID(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +/* TODO: remove these five (so far used in action.c) */ +char *getMSG(msg_t *pM); +char *getHOSTNAME(msg_t *pM); +char *getPROCID(msg_t *pM); +char *getAPPNAME(msg_t *pM); +int getMSGLen(msg_t *pM); + +char *getHOSTNAME(msg_t *pM); +int getHOSTNAMELen(msg_t *pM); +char *getProgramName(msg_t *pM); +int getProgramNameLen(msg_t *pM); +uchar *getRcvFrom(msg_t *pM); + +#if 0 +char *getUxTradMsg(msg_t *pM); +#endif + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains * us quite some performance and is fully abstracted using this header file. diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 914c2f2c..78829f94 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -280,7 +280,7 @@ rsRetVal objName##ClassExit(void) \ * rgerhards, 2008-01-30 */ #define BEGINobjDestruct(OBJ) \ - rsRetVal OBJ##Destruct(OBJ##_t **ppThis) \ + rsRetVal OBJ##Destruct(OBJ##_t __attribute__((unused)) **ppThis) \ { \ DEFiRet; \ int iCancelStateSave; \ @@ -314,7 +314,7 @@ rsRetVal objName##ClassExit(void) \ #define PROTOTYPEObjDebugPrint(obj) rsRetVal obj##DebugPrint(obj##_t *pThis) #define INTERFACEObjDebugPrint(obj) rsRetVal (*DebugPrint)(obj##_t *pThis) #define BEGINobjDebugPrint(obj) \ - rsRetVal obj##DebugPrint(obj##_t *pThis) \ + rsRetVal obj##DebugPrint(obj##_t __attribute__((unused)) *pThis) \ { \ DEFiRet; \ diff --git a/runtime/parser.c b/runtime/parser.c index 212d40f3..64e03094 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -191,6 +191,23 @@ sanitizeMessage(msg_t *pMsg) lenMsg--; } + /* it is much quicker to sweep over the message and see if it actually + * needs sanitation than to do the sanitation in any case. So we first do + * this and terminate when it is not needed - which is expectedly the case + * for the vast majority of messages. -- rgerhards, 2009-06-15 + */ + int bNeedSanitize = 0; + for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { + if(pszMsg[iSrc] < 32) { + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + bNeedSanitize = 1; + break; + } + } + } + if(bNeedSanitize == 0) + FINALIZE; + /* now copy over the message and sanitize it */ /* TODO: can we get cheaper memory alloc? {alloca()?}*/ iMaxLine = glbl.GetMaxLine(); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ea303a51..32177a9f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -103,6 +103,8 @@ typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif +typedef char bool; /* I intentionally use char, to keep it slim so that many fit into the CPU cache! */ + /* settings for flow control * TODO: is there a better place for them? -- rgerhards, 2008-03-14 */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 07256fab..f3d9aa48 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -299,10 +299,8 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->pBuf != NULL) - free(pThis->pBuf); - if(pThis->pszBuf != NULL) - free(pThis->pszBuf); + free(pThis->pBuf); + free(pThis->pszBuf); if(pszNew == NULL) { pThis->iStrLen = 0; pThis->iBufSize = 0; diff --git a/tcps_sess.c b/tcps_sess.c index 62d51f66..cfee0523 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -97,12 +97,9 @@ CODESTARTobjDestruct(tcps_sess) pThis->pSrv->pOnSessDestruct(&pThis->pUsr); } /* now destruct our own properties */ - if(pThis->fromHost != NULL) - free(pThis->fromHost); - if(pThis->fromHostIP != NULL) - free(pThis->fromHostIP); - if(pThis->pMsg != NULL) - free(pThis->pMsg); + free(pThis->fromHost); + free(pThis->fromHostIP); + free(pThis->pMsg); ENDobjDestruct(tcps_sess) @@ -124,10 +121,7 @@ SetHost(tcps_sess_t *pThis, uchar *pszHost) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHost != NULL) { - free(pThis->fromHost); - } - + free(pThis->fromHost); pThis->fromHost = pszHost; RETiRet; @@ -144,10 +138,7 @@ SetHostIP(tcps_sess_t *pThis, uchar *pszHostIP) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHostIP != NULL) { - free(pThis->fromHostIP); - } - + free(pThis->fromHostIP); pThis->fromHostIP = pszHostIP; RETiRet; @@ -226,11 +217,9 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis) +defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime) { msg_t *pMsg; - struct syslogTime stTime; - time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -240,16 +229,13 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } - //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); - //} /* we now create our own message object and submit it to the queue */ - CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); pMsg->iLenRawMsg = pThis->iMsg; - MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); + MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName, pThis->pLstnInfo->lenInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; @@ -281,6 +267,8 @@ finalize_it: static rsRetVal PrepareClose(tcps_sess_t *pThis) { + struct syslogTime stTime; + time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -306,7 +294,8 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - defaultDoSubmitMessage(pThis); + datetime.getCurrTime(&stTime, &ttGenTime); + defaultDoSubmitMessage(pThis, &stTime, ttGenTime); } finalize_it: @@ -341,7 +330,7 @@ Close(tcps_sess_t *pThis) * rgerhards, 2008-03-14 */ static rsRetVal -processDataRcvd(tcps_sess_t *pThis, char c) +processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -387,7 +376,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -398,7 +387,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -415,7 +404,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); pThis->inputState = eAtStrtFram; } } @@ -436,22 +425,29 @@ processDataRcvd(tcps_sess_t *pThis, char c) * RS_RET_OK, which means the session should be kept open * or anything else, which means it must be closed. * rgerhards, 2008-03-01 + * As a performance optimization, we pick up the timestamp here. Acutally, + * this *is* the *correct* reception step for all the data we received, because + * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { - DEFiRet; + struct syslogTime stTime; + time_t ttGenTime; char *pEnd; + DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); assert(pData != NULL); assert(iLen > 0); + datetime.getCurrTime(&stTime, &ttGenTime); + /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++)); + CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime)); } finalize_it: diff --git a/tcpsrv.c b/tcpsrv.c index 3516b2e3..11619498 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -105,6 +105,7 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) pEntry->pszPort = pszPort; pEntry->pSrv = pThis; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + pEntry->lenInputName = ustrlen(pEntry->pszInputName); /* and add to list */ pEntry->pNext = pThis->pLstnPorts; @@ -510,7 +511,7 @@ Run(tcpsrv_t *pThis) while(nfds && iTCPSess != -1) { CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); /* Receive message */ diff --git a/tcpsrv.h b/tcpsrv.h index 2d174ce0..95537d54 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -37,6 +37,7 @@ typedef enum ETCPsyslogFramingAnomaly { struct tcpLstnPortList_s { uchar *pszPort; /**< the ports the listener shall listen on */ uchar *pszInputName; /**< value to be used as input name */ + size_t lenInputName; /**< length of inputName */ tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ tcpLstnPortList_t *pNext; /**< next port or NULL */ }; diff --git a/tools/syslogd.c b/tools/syslogd.c index b43b7a37..a9f1dbdb 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -639,7 +639,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) - MsgSetInputName(pMsg, pszInputName); + MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName)); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsg(pMsg, (char*)msg); @@ -947,7 +947,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); -- cgit From f7579e68a67364c8040966be57c2eae4c9550ee5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 11:36:05 +0200 Subject: done various optimizations to the stringbuf and its users --- outchannel.c | 20 ++--- parse.c | 16 ++-- runtime/cfsysline.c | 12 +-- runtime/conf.c | 40 +++++----- runtime/ctok.c | 4 +- runtime/datetime.c | 5 +- runtime/msg.c | 30 ++----- runtime/msg.h | 4 +- runtime/nsd_gtls.c | 8 +- runtime/obj.c | 6 +- runtime/parser.c | 19 ++++- runtime/stream.c | 2 +- runtime/stringbuf.c | 221 ++++++++++++++++++---------------------------------- runtime/stringbuf.h | 27 +++---- runtime/vm.c | 2 +- runtime/vmop.c | 2 +- template.c | 54 ++++++------- tools/syslogd.c | 170 ++++++++++++++++++++-------------------- 18 files changed, 282 insertions(+), 360 deletions(-) diff --git a/outchannel.c b/outchannel.c index 5c348b63..4f8abb32 100644 --- a/outchannel.c +++ b/outchannel.c @@ -105,22 +105,22 @@ static rsRetVal get_Field(uchar **pp, uchar **pField) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p && *p != ' ' && *p != ',') { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pField, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pField, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -174,22 +174,22 @@ static inline rsRetVal get_restOfLine(uchar **pp, uchar **pBuf) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p) { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pBuf, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pBuf, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; diff --git a/parse.c b/parse.c index 58458d62..87c67676 100644 --- a/parse.c +++ b/parse.c @@ -268,7 +268,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim /* We got the string, now take it and see if we need to * remove anything at its end. */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); if(bTrimTrailing) { CHKiRet(rsCStrTrimTrailingWhiteSpace(pCStr)); @@ -344,7 +344,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) } /* We got the string, let's finish it... */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* done! */ *ppCStr = pCStr; @@ -380,7 +380,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) assert(pIP != NULL); assert(pBits != NULL); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); parsSkipWhitespace(pThis); pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; @@ -390,8 +390,8 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) */ while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != '/' && *pC != ',' && !isspace((int)*pC)) { - if((iRet = rsCStrAppendChar(pCStr, *pC)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrAppendChar(pCStr, *pC)) != RS_RET_OK) { + cstrDestruct (&pCStr); FINALIZE; } ++pThis->iCurrPos; @@ -399,15 +399,15 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) } /* We got the string, let's finish it... */ - if((iRet = rsCStrFinish(pCStr)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrFinalize(pCStr)) != RS_RET_OK) { + cstrDestruct(&pCStr); FINALIZE; } /* now we have the string and must check/convert it to * an NetAddr structure. */ - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pszIP, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pszIP, 0)); *pIP = calloc(1, sizeof(struct NetAddr)); diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index e1e4a6a4..0fb4247d 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -462,7 +462,7 @@ getWord(uchar **pp, cstr_t **ppStrB) ASSERT(*pp != NULL); ASSERT(ppStrB != NULL); - CHKiRet(rsCStrConstruct(ppStrB)); + CHKiRet(cstrConstruct(ppStrB)); skipWhiteSpace(pp); /* skip over any whitespace */ @@ -470,9 +470,9 @@ getWord(uchar **pp, cstr_t **ppStrB) p = *pp; while(*p && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(*ppStrB, *p++)); + CHKiRet(cstrAppendChar(*ppStrB, *p++)); } - CHKiRet(rsCStrFinish(*ppStrB)); + CHKiRet(cstrFinalize(*ppStrB)); *pp = p; @@ -506,7 +506,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); pStrB = NULL; /* we got the word, now set it */ @@ -525,7 +525,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -548,7 +548,7 @@ doSyslogName(uchar **pp, rsRetVal (*pSetHdlr)(void*, int), void *pVal, syslogNam ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); /* get word */ - iNewVal = decodeSyslogName(rsCStrGetSzStr(pStrB), pNameTable); + iNewVal = decodeSyslogName(cstrGetSzStr(pStrB), pNameTable); if(pSetHdlr == NULL) { /* we should set value directly to var */ diff --git a/runtime/conf.c b/runtime/conf.c index 7cdcf5ec..65fc66f1 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -526,17 +526,15 @@ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEn tplName = (uchar*) strdup((char*)dfltTplName); } else { /* template specified, pick it up */ - if(rsCStrConstruct(&pStrB) != RS_RET_OK) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKiRet(cstrConstruct(&pStrB)); /* now copy the string */ while(*p && *p != '#' && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &tplName, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &tplName, 0)); } iRet = OMSRsetEntry(pOMSR, iEntry, tplName, iTplOpts); @@ -925,7 +923,7 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) */ static rsRetVal cflineProcessHostSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -951,21 +949,20 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) dbgprintf("resetting BSD-like hostname filter\n"); eDfltHostnameCmpMode = HN_NO_COMP; if(pDfltHostnameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, NULL)); } } else { dbgprintf("setting BSD-like hostname filter to '%s'\n", *pline); if(pDfltHostnameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } @@ -976,7 +973,7 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) */ static rsRetVal cflineProcessTagSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -995,21 +992,20 @@ static rsRetVal cflineProcessTagSelector(uchar **pline) if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { dbgprintf("resetting programname filter\n"); if(pDfltProgNameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, NULL)); } } else { dbgprintf("setting programname filter to '%s'\n", *pline); if(pDfltProgNameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } diff --git a/runtime/ctok.c b/runtime/ctok.c index d2cd8bbd..263e656c 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -269,7 +269,7 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ - CHKiRet(rsCStrFinish(pstrVal)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -319,7 +319,7 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(ctokGetCharFromStream(pThis, &c)); } - CHKiRet(rsCStrFinish(pStrB)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; diff --git a/runtime/datetime.c b/runtime/datetime.c index 19e61a0a..40ab4e9c 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -152,11 +152,10 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) static int srSLMGParseInt32(uchar** ppsz) { - int i; + register int i; i = 0; - while(isdigit((int) **ppsz)) - { + while(isdigit((int) **ppsz)) { i = i * 10 + **ppsz - '0'; ++(*ppsz); } diff --git a/runtime/msg.c b/runtime/msg.c index b3c76089..55cc48b8 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -437,11 +437,12 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); + pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); tmpCOPYSZ(SeverityStr); tmpCOPYSZ(Facility); tmpCOPYSZ(FacilityStr); - tmpCOPYSZ(PRI); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); tmpCOPYSZ(UxTradMsg); @@ -587,7 +588,7 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) } /* OK, finaally we could obtain a PROCID. So let's use it ;) */ - CHKiRet(rsCStrFinish(pM->pCSPROCID)); + CHKiRet(cstrFinalize(pM->pCSPROCID)); finalize_it: RETiRet; @@ -629,7 +630,7 @@ static rsRetVal aquireProgramName(msg_t *pM) ; ++i) { CHKiRet(rsCStrAppendChar(pM->pCSProgName, pM->pszTAG[i])); } - CHKiRet(rsCStrFinish(pM->pCSProgName)); + CHKiRet(cstrFinalize(pM->pCSProgName)); } finalize_it: RETiRet; @@ -724,31 +725,12 @@ char *getMSG(msg_t *pM) /* Get PRI value in text form */ -static char *getPRI(msg_t *pM) +static inline char *getPRI(msg_t *pM) { - int pri; - BEGINfunc - if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszPRI == NULL) { - /* OK, we need to construct it... we use a 5 byte buffer - as of - * RFC 3164, it can't be longer. Should it still be, snprintf will truncate... - * Note that we do not use the LOG_MAKEPRI macro. This macro - * is a simple add of the two values under FreeBSD 7. So we implement - * the logic in our own code. This is a change from a bug - * report. -- rgerhards, 2008-07-14 - */ - pri = pM->iFacility * 8 + pM->iSeverity; - pM->pszPRI = pM->bufPRI; - pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); - } - MsgUnlock(pM); - - ENDfunc - return (char*)pM->pszPRI; + return (char*)pM->bufPRI; } diff --git a/runtime/msg.h b/runtime/msg.h index 1689bbbc..20360641 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -73,7 +73,8 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - uchar *pszPRI; /* the PRI as a string */ + //uchar *pszPRI; /* the PRI as a string */ + uchar bufPRI[5]; int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we @@ -121,7 +122,6 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ /* now follow fixed-size buffers to safe some time otherwise used for allocs */ - uchar bufPRI[5]; }; diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 1a50e2f8..19dc8678 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -335,7 +335,7 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) gnutls_x509_crt_deinit(cert); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; finalize_it: @@ -455,7 +455,7 @@ GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) snprintf((char*)buf, sizeof(buf), ":%2.2X", pFingerprint[i]); CHKiRet(rsCStrAppendStrWithLen(pStr, buf, 3)); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -723,7 +723,7 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) } ++i; /* char processed */ } - CHKiRet(rsCStrFinish(pstrCN)); + CHKiRet(cstrFinalize(pstrCN)); /* we got it - we ignore the rest of the DN string (if any). So we may * not detect if it contains more than one CN @@ -884,7 +884,7 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) if(!bFoundPositiveMatch) { dbgprintf("invalid peer name, not permitted to talk to it\n"); if(pThis->bReportAuthErr == 1) { - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer name not authorized - " "not permitted to talk to it. Names: %s", diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..355c0f97 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -447,7 +447,7 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -515,7 +515,7 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pCStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* check terminator */ if(c != ':') ABORT_FINALIZE(RS_RET_INVALID_DELIMITER); @@ -629,7 +629,7 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pProp->pcsName, c)); NEXTC; } - CHKiRet(rsCStrFinish(pProp->pcsName)); + CHKiRet(cstrFinalize(pProp->pcsName)); /* property type */ CHKiRet(objDeserializeNumber(&i, pStrm)); diff --git a/runtime/parser.c b/runtime/parser.c index 64e03094..13fb51ec 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -205,8 +205,16 @@ sanitizeMessage(msg_t *pMsg) } } } - if(bNeedSanitize == 0) + if(bNeedSanitize == 0) { + /* what a shame - we do not have a \0 byte... + * TODO: think about adding it or otherwise be able to use it... + */ + uchar *pRaw; + CHKmalloc(pRaw = realloc(pMsg->pszRawMsg, pMsg->iLenRawMsg + 1)); + pRaw[pMsg->iLenRawMsg] = '\0'; + pMsg->pszRawMsg = pRaw; FINALIZE; + } /* now copy over the message and sanitize it */ /* TODO: can we get cheaper memory alloc? {alloca()?}*/ @@ -276,6 +284,7 @@ rsRetVal parseMsg(msg_t *pMsg) DEFiRet; uchar *msg; int pri; + int iPriText; CHKiRet(sanitizeMessage(pMsg)); @@ -285,11 +294,19 @@ rsRetVal parseMsg(msg_t *pMsg) /* pull PRI */ pri = DEFUPRI; msg = pMsg->pszRawMsg; + iPriText = 0; if(*msg == '<') { + /* while we process the PRI, we also fill the PRI textual representation + * inside the msg object. This may not be ideal from an OOP point of view, + * but it offers us performance... + */ pri = 0; while(isdigit((int) *++msg)) { + pMsg->bufPRI[iPriText++ % 4] = *msg; /* mod 4 to guard against malformed messages! */ pri = 10 * pri + (*msg - '0'); } + pMsg->bufPRI[iPriText % 4] = '\0'; + pMsg->iLenPRI = iPriText % 4; if(*msg == '>') ++msg; if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..1cff2da6 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -368,7 +368,7 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr) CHKiRet(rsCStrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } - CHKiRet(rsCStrFinish(*ppCStr)); + CHKiRet(cstrFinalize(*ppCStr)); finalize_it: if(iRet != RS_RET_OK && *ppCStr != NULL) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index f3d9aa48..a2d9c599 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -6,8 +6,9 @@ * Please see syslogd.c for license information. * All functions in this "class" start with rsCStr (rsyslog Counted String). * begun 2005-09-07 rgerhards + * did some optimization (read: bugs!) rgerhards, 2009-06-16 * - * Copyright (C) 2007-2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2007-2009 by Rainer Gerhards and Adiscon GmbH * * This file is part of the rsyslog runtime library. * @@ -40,6 +41,7 @@ #include "regexp.h" #include "obj.h" +uchar* rsCStrGetSzStr(cstr_t *pThis); /* ################################################################# * * private members * @@ -54,15 +56,14 @@ DEFobjCurrIf(regexp) * ################################################################# */ -rsRetVal rsCStrConstruct(cstr_t **ppThis) +rsRetVal cstrConstruct(cstr_t **ppThis) { DEFiRet; cstr_t *pThis; ASSERT(ppThis != NULL); - if((pThis = (cstr_t*) calloc(1, sizeof(cstr_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pThis = (cstr_t*) calloc(1, sizeof(cstr_t))); rsSETOBJTYPE(pThis, OIDrsCStr); pThis->pBuf = NULL; @@ -89,7 +90,7 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); - pThis->iBufSize = pThis->iStrLen = strlen((char*)(char *) sz); + pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -137,24 +138,8 @@ void rsCStrDestruct(cstr_t **ppThis) { cstr_t *pThis = *ppThis; - /* rgerhards 2005-10-19: The free of pBuf was contained in conditional compilation. - * The code was only compiled if STRINGBUF_TRIM_ALLOCSIZE was set to 1. I honestly - * do not know why it was so, I think it was an artifact. Anyhow, I have changed this - * now. Should there any issue occur, this comment hopefully will shed some light - * on what happened. I re-verified, and this function has never before been called - * by anyone. So changing it can have no impact for obvious reasons... - * - * rgerhards, 2008-02-20: I changed the interface to the new calling conventions, where - * the destructor receives a pointer to the object, so that it can set it to NULL. - */ - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } - - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - } - + free(pThis->pBuf); + free(pThis->pszBuf); RSFREEOBJ(pThis); *ppThis = NULL; } @@ -166,12 +151,14 @@ void rsCStrDestruct(cstr_t **ppThis) * allocated. In practice, a bit more is allocated because we envision that * some more characters may be added after these. * rgerhards, 2008-01-07 + * changed to utilized realloc() -- rgerhards, 2009-06-16 */ -static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) +static rsRetVal +rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { - DEFiRet; uchar *pNewBuf; size_t iNewSize; + DEFiRet; /* first compute the new size needed */ if(iMinNeeded > pThis->iAllocIncrement) { @@ -187,15 +174,9 @@ static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) } iNewSize += pThis->iBufSize; /* add current size */ - /* and then allocate and copy over */ /* DEV debugging only: dbgprintf("extending string buffer, old %d, new %d\n", pThis->iBufSize, iNewSize); */ - if((pNewBuf = (uchar*) malloc(iNewSize * sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - memcpy(pNewBuf, pThis->pBuf, pThis->iBufSize); + CHKmalloc(pNewBuf = (uchar*) realloc(pThis->pBuf, iNewSize * sizeof(uchar))); pThis->iBufSize = iNewSize; - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } pThis->pBuf = pNewBuf; finalize_it: @@ -288,6 +269,29 @@ finalize_it: } +/* NEW VARIANT + * Append a character to the current string object. This may only be done until + * cstrFinalize() is called. + * rgerhards, 2009-06-16 + */ +rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) +{ + DEFiRet; + + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + if(pThis->iStrLen >= pThis->iBufSize) { + CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ + } + + /* ok, when we reach this, we have sufficient memory */ + *(pThis->pBuf + pThis->iStrLen++) = c; + +finalize_it: + RETiRet; +} + + /* Sets the string object to the classigal sz-string provided. * Any previously stored vlaue is discarded. If a NULL pointer * the the new value (pszNew) is provided, an empty string is @@ -388,35 +392,42 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) } +/* NEW VERSION for interface without separate psz buffer! */ +/* Returns the cstr data as a classical C sz string. We use that the + * Finalizer did properly terminate our string (but we may stil be NULL). + * So it is vital that the finalizer is called BEFORe this function here! + * The caller must not free or otherwise manipulate the returned string and must not + * destroy the CStr object as long as the ascii string is used. + * This function may return NULL, if the string is currently NULL. This + * is a feature, not a bug. If you need non-NULL in any case, use + * cstrGetSzStrNoNULL() instead. + * Note that due to the new single-buffer interface this function almost does nothing! + * rgerhards, 2006-09-16 + */ +uchar* cstrGetSzStr(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + return(pThis->pBuf); +} + + /* Converts the CStr object to a classical zero-terminated C string, * returns that string and destroys the CStr object. The returned string * MUST be freed by the caller. The function might return NULL if * no memory can be allocated. * - * TODO: - * This function should at some time become special. The base idea is to - * add one extra byte to the end of the regular buffer, so that we can - * convert it to an szString without the need to copy. The extra memory - * footprint is not hefty, but the performance gain is potentially large. - * To get it done now, I am not doing the optimiziation right now. - * rgerhards, 2005-09-07 + * This is the NEW replacement for rsCStrConvSzStrAndDestruct which does + * no longer utilize a special buffer but soley works on pBuf (and also + * assumes that cstrFinalize had been called). * - * rgerhards, 2007-09-04: I have changed the interface of this function. It now - * returns an rsRetVal, so that we can communicate back if we have an error. - * Using the standard method is much better than returning NULL. Secondly, NULL - * was not actually an error - it was in indication if the string was empty. - * This was needed in some parts of the code, in others not. I have now added - * a second parameter to specify what the caller needs. I hope these changes - * will make it less likely that the function is called incorrectly, what - * previously happend quite often and was the cause of a number of program - * aborts. So the parameters are now: + * Parameters are as follows: * pointer to the object, pointer to string-pointer to receive string and * bRetNULL: 0 - must not return NULL on empty string, return "" in that * case, 1 - return NULL instead of an empty string. * PLEASE NOTE: the caller must free the memory returned in ppSz in any case * (except, of course, if it is NULL). */ -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) { DEFiRet; uchar* pRetBuf; @@ -427,14 +438,13 @@ rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - if((pRetBuf = malloc(sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pRetBuf = malloc(sizeof(uchar))); *pRetBuf = '\0'; } else { pRetBuf = NULL; } } else - pRetBuf = rsCStrGetSzStr(pThis); + pRetBuf = pThis->pBuf; *ppSz = pRetBuf; @@ -443,60 +453,39 @@ finalize_it: * that we can NOT use the rsCStrDestruct function as it would * also free the sz String buffer, which we pass on to the user. */ - if(pThis->pBuf != NULL) - free(pThis->pBuf); RSFREEOBJ(pThis); - RETiRet; } -#if STRINGBUF_TRIM_ALLOCSIZE == 1 - /* Only in this mode, we need to trim the string. To do - * so, we must allocate a new buffer of the exact - * string size, and then copy the old one over. - */ - /* WARNING - * STRINGBUF_TRIM_ALLOCSIZE can, in theory, be used to trim - * memory buffers. This part of the code was inherited from - * liblogging (where it is used in a different context) but - * never put to use in rsyslog. The reason is that it is hardly - * imaginable where the extra performance cost is worth the save - * in memory alloc. Then Anders Blomdel rightfully pointed out that - * the code does not work at all - and nobody even know that it - * probably shouldn't. Rather than removing, I deciced to somewhat - * fix the code, so that this feature may be enabled if somebody - * really has a need for it. Be warned, however, that I NEVER - * tested the fix. So if you intend to use this feature, you must - * do full testing before you rely on it. -- rgerhards, 2008-02-12 - */ -rsRetVal rsCStrFinish(cstr_t __attribute__((unused)) *pThis) +/* Finalize the string object. This must be called after all data is added to it + * but before that data is used. + * rgerhards, 2009-06-16 + */ +rsRetVal +cstrFinalize(cstr_t *pThis) { DEFiRet; - uchar* pBuf; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + assert(pThis->bIsFinalized == 0); - if((pBuf = malloc((pThis->iStrLen) * sizeof(uchar))) == NULL) - { /* OK, in this case we use the previous buffer. At least - * we have it ;) - */ - } - else - { /* got the new buffer, so let's use it */ - memcpy(pBuf, pThis->pBuf, pThis->iStrLen); - pThis->pBuf = pBuf; + if(pThis->iStrLen > 0) { + /* terminate string only if one exists */ + CHKiRet(cstrAppendChar(pThis, '\0')); + --pThis->iStrLen; /* do NOT count the \0 byte */ } + pThis->bIsFinalized = 1; +finalize_it: RETiRet; } -#endif /* #if STRINGBUF_TRIM_ALLOCSIZE == 1 */ void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); assert(iNewIncrement > 0); - pThis->iAllocIncrement = iNewIncrement; } @@ -1025,56 +1014,6 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz) } -#if 0 /* read comment below why this is commented out. In short: for future use! */ -/* locate the first occurence of a standard sz string inside a rsCStr object. - * Returns the offset (0-bound) of this first occurrence. If not found, -1 is - * returned. - * rgerhards 2005-09-19 - * WARNING: I accidently created this function (I later noticed I didn't relly - * need it... I will not remove the function, as it probably is useful - * some time later. However, it is not fully tested, so start with testing - * it before you put it to first use). - */ -int rsCStrLocateSzStr(cstr_t *pThis, uchar *sz) -{ - int iLenSz; - int i; - int iMax; - int bFound; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(sz == NULL) - return 0; - - iLenSz = strlen((char*)sz); - if(iLenSz == 0) - return 0; - - /* compute the largest index where a match could occur - after all, - * the to-be-located string must be able to be present in the - * searched string (it needs its size ;)). - */ - iMax = pThis->iStrLen - iLenSz; - - bFound = 0; - i = 0; - while(i < iMax && !bFound) { - int iCheck; - uchar *pComp = pThis->pBuf + i; - for(iCheck = 0 ; iCheck < iLenSz ; ++iCheck) - if(*(pComp + iCheck) != *(sz + iCheck)) - break; - if(iCheck == iLenSz) - bFound = 1; /* found! - else it wouldn't be equal */ - else - ++i; /* on to the next try */ - } - - return(bFound ? i : -1); -} -#endif /* end comment out */ - - /* our exit function. TODO: remove once converted to a class * rgerhards, 2008-03-11 */ @@ -1098,11 +1037,5 @@ finalize_it: } -/* - * Local variables: - * c-indent-level: 8 - * c-basic-offset: 8 - * tab-width: 8 - * End: - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 684133bb..d28aee26 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -49,13 +49,15 @@ typedef struct cstr_s size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ size_t iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ + bool bIsFinalized; /**< is this object finished and ready for use? (a debug aid, may be removed later TODO 2009-06-16) */ } cstr_t; /** * Construct a rsCStr object. */ -rsRetVal rsCStrConstruct(cstr_t **ppThis); +rsRetVal cstrConstruct(cstr_t **ppThis); +#define rsCStrConstruct(x) cstrConstruct((x)) rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz); rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); @@ -63,6 +65,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); * Destruct the string buffer object. */ void rsCStrDestruct(cstr_t **ppThis); +#define cstrDestruct(x) rsCStrDestruct((x)) /** * Append a character to an existing string. If necessary, the @@ -71,6 +74,7 @@ void rsCStrDestruct(cstr_t **ppThis); * \param c Character to append to string. */ rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); +rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); /** * Truncate "n" number of characters from the end of the @@ -123,10 +127,9 @@ rsRetVal rsCStrAppendInt(cstr_t *pThis, long i); rsRetVal strExit(void); /* TODO: remove once we have a real object interface! */ -uchar* rsCStrGetSzStr(cstr_t *pThis); +uchar* __attribute__((deprecated)) rsCStrGetSzStr(cstr_t *pThis); uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis); rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew); -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); int rsCStrCStrCmp(cstr_t *pCS1, cstr_t *pCS2); int rsCStrSzStrCmp(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrOffsetSzStrCmp(cstr_t *pCS1, size_t iOffset, uchar *psz, size_t iLenSz); @@ -142,6 +145,11 @@ rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); +/* new calling interface */ +rsRetVal cstrFinalize(cstr_t *pThis); +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); +uchar* cstrGetSzStr(cstr_t *pThis); + /* now come inline-like functions */ #ifdef NDEBUG # define rsCStrLen(x) ((int)((x)->iStrLen)) @@ -149,19 +157,6 @@ rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); int rsCStrLen(cstr_t *pThis); #endif -#if STRINGBUF_TRIM_ALLOCSIZE != 1 -/* This is the normal case (see comment in rsCStrFinish!). In those cases, the function - * simply needs to do nothing, so that we can save us the function call. - * rgerhards, 2008-02-12 - */ -# define rsCStrFinish(pThis) RS_RET_OK -#else - /** - * Finish the string buffer dynamic allocation. - */ - rsRetVal rsCStrFinish(cstr_t *pThis); -#endif - #define rsCStrGetBufBeg(x) ((x)->pBuf) rsRetVal strInit(); diff --git a/runtime/vm.c b/runtime/vm.c index 125b0d21..8cbf9e12 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -563,7 +563,7 @@ rsf_tolower(vmstk_t *pStk, int numOperands) } /* Store result and cleanup */ - CHKiRet(rsCStrFinish(pcstr)); + CHKiRet(cstrFinalize(pcstr)); var.SetString(operand1, pcstr); vmstk.Push(pStk, operand1); finalize_it: diff --git a/runtime/vmop.c b/runtime/vmop.c index 3e001d27..acacfc9e 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -85,7 +85,7 @@ CODESTARTobjDebugPrint(vmop) CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); } } - CHKiRet(rsCStrFinish(&pStrVar)); + CHKiRet(cstrFinalize(pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); if(pThis->opcode != opcode_FUNC_CALL) rsCStrDestruct(&pStrVar); diff --git a/template.c b/template.c index 4a12aa3b..6e34bcd8 100644 --- a/template.c +++ b/template.c @@ -88,7 +88,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) * free the obtained value (if requested). We continue this * loop until we got hold of all values. */ - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); pTpe = pTpl->pEntryRoot; while(pTpe != NULL) { @@ -99,7 +99,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) ) { dbgprintf("error %d during tplToString()\n", iRet); /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); FINALIZE; } } else if(pTpe->eEntryType == FIELD) { @@ -119,7 +119,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, (uchar*) pVal, iLenVal)) { dbgprintf("error %d during tplToString()\n", iRet); /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); if(bMustBeFreed) free(pVal); FINALIZE; @@ -133,8 +133,8 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) /* we are done with the template, now let's convert the result into a * "real" (usable) string and discard the helper structures. */ - CHKiRet(rsCStrFinish(pCStr)); - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pVal, 0)); + CHKiRet(cstrFinalize(pCStr)); + CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pVal, 0)); finalize_it: *ppSz = (iRet == RS_RET_OK) ? pVal : NULL; @@ -272,21 +272,21 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM p = *pp; iLen = *pLen; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); while(*p) { if(*p == '\'') { - CHKiRet(rsCStrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); iLen++; /* reflect the extra character */ } else if((escapeMode == 1) && (*p == '\\')) { - CHKiRet(rsCStrAppendChar(pStrB, '\\')); + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); if(*pbMustBeFreed) free(*pp); /* discard previous value */ @@ -299,7 +299,7 @@ finalize_it: if(iRet != RS_RET_OK) { doSQLEmergencyEscape(*pp, escapeMode); if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -376,7 +376,7 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) p = *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; rsCStrSetAllocIncrement(pStrB, 32); /* process the message and expand escapes @@ -387,22 +387,22 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) switch(*++p) { case '\0': /* the best we can do - it's invalid anyhow... */ - rsCStrAppendChar(pStrB, *p); + cstrAppendChar(pStrB, *p); break; case 'n': - rsCStrAppendChar(pStrB, '\n'); + cstrAppendChar(pStrB, '\n'); ++p; break; case 'r': - rsCStrAppendChar(pStrB, '\r'); + cstrAppendChar(pStrB, '\r'); ++p; break; case '\\': - rsCStrAppendChar(pStrB, '\\'); + cstrAppendChar(pStrB, '\\'); ++p; break; case '%': - rsCStrAppendChar(pStrB, '%'); + cstrAppendChar(pStrB, '%'); ++p; break; case '0': /* numerical escape sequence */ @@ -419,15 +419,15 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) while(*p && isdigit((int)*p)) { i = i * 10 + *p++ - '0'; } - rsCStrAppendChar(pStrB, i); + cstrAppendChar(pStrB, i); break; default: - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); break; } } else - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); } if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -438,14 +438,14 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) return 1; } pTpe->eEntryType = CONSTANT; - rsCStrFinish(pStrB); + cstrFinalize(pStrB); /* We obtain the length from the counted string object * (before we delete it). Later we might take additional * benefit from the counted string object. * 2005-09-09 rgerhards */ pTpe->data.constant.iLenConstant = rsCStrLen(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) + if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) return 1; *pp = p; @@ -552,7 +552,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) p = (unsigned char*) *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -563,13 +563,13 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) pTpe->eEntryType = FIELD; while(*p && *p != '%' && *p != ':') { - rsCStrAppendChar(pStrB, tolower(*p)); + cstrAppendChar(pStrB, tolower(*p)); ++p; /* do NOT do this in tolower()! */ } /* got the name*/ - rsCStrFinish(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) + cstrFinalize(pStrB); + if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) return 1; /* Check frompos, if it has an R, then topos should be a regex */ diff --git a/tools/syslogd.c b/tools/syslogd.c index a9f1dbdb..c224087d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -553,7 +553,7 @@ static char **crunch_list(char *list) #if 0 count=0; while (result[count]) - dbgprintf("#%d: %s\n", count, StripDomains[count++]); + DBGPRINTF("#%d: %s\n", count, StripDomains[count++]); #endif return result; } @@ -780,7 +780,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * rgerhards, 2007-09-14 */ if(*(msg + len - 1) == '\0') { - dbgprintf("dropped NUL at very end of message\n"); + DBGPRINTF("dropped NUL at very end of message\n"); len--; } @@ -790,7 +790,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * turn on/off this handling. rgerhards, 2007-07-23 */ if(bDropTrailingLF && *(msg + len - 1) == '\n') { - dbgprintf("dropped LF at very end of message (DropTrailingLF is set)\n"); + DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); len--; } @@ -815,7 +815,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla iLenDefBuf = iMaxLine; CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); - dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", + DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, len-1); /* Now check if the uncompression worked. If not, there is not much we can do. In * that case, we log an error message but ignore the message itself. Storing the @@ -861,7 +861,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * (I couldn't do any more smart things anyway...). * rgerhards, 2007-9-20 */ - dbgprintf("internal error: iMsg > max msg size in printchopped()\n"); + DBGPRINTF("internal error: iMsg > max msg size in printchopped()\n"); } FINALIZE; /* in this case, we are done... nothing left we can do */ } @@ -1025,14 +1025,14 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { /* not equal, so we are already done... */ - dbgprintf("hostname filter '+%s' does not match '%s'\n", + DBGPRINTF("hostname filter '+%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); FINALIZE; } } else { /* must be -hostname */ if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { /* not equal, so we are already done... */ - dbgprintf("hostname filter '-%s' does not match '%s'\n", + DBGPRINTF("hostname filter '-%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); FINALIZE; } @@ -1053,7 +1053,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce if((!bEqv && !bInv) || (bEqv && bInv)) { /* not equal or inverted selection, so we are already done... */ - dbgprintf("programname filter '%s' does not match '%s'\n", + DBGPRINTF("programname filter '%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); FINALIZE; } @@ -1063,7 +1063,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce if(f->f_filter_type == FILTER_PRI) { /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ + if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) bRet = 0; else @@ -1074,7 +1074,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce CHKiRet(vm.SetMsg(pVM, pMsg)); CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + DBGPRINTF("result of expression evaluation: %lld\n", pResult->val.num); /* VM is destructed on function exit */ bRet = (pResult->val.num) ? 1 : 0; } else { @@ -1166,7 +1166,7 @@ DEFFUNC_llExecFunc(processMsgDoActions) assert(pAction != NULL); if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - dbgprintf("not calling action because the previous one is not suspended\n"); + DBGPRINTF("not calling action because the previous one is not suspended\n"); ABORT_FINALIZE(RS_RET_OK); } @@ -1399,7 +1399,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); } } else { - dbgprintf("no TIMESTAMP detected!\n"); + DBGPRINTF("no TIMESTAMP detected!\n"); bContParse = 0; } @@ -1561,7 +1561,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) /* indeed, this smells like a TAG, so lets use it for this. We take * the HOSTNAME from the sender system instead. */ - dbgprintf("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); + DBGPRINTF("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); moveHOSTNAMEtoTAG(pMsg); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } @@ -1586,22 +1586,21 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the records: the code is currently clean, but we could optimize it! */ if(!bTAGCharDetected) { uchar *pszTAG; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; rsCStrSetAllocIncrement(pStrB, 33); pWork = pBuf; iCnt = 0; while(*p2parse && *p2parse != ':' && *p2parse != ' ') { - rsCStrAppendChar(pStrB, *p2parse++); + cstrAppendChar(pStrB, *p2parse++); ++iCnt; } if(*p2parse == ':') { ++p2parse; - rsCStrAppendChar(pStrB, ':'); + cstrAppendChar(pStrB, ':'); } - rsCStrFinish(pStrB); - - rsCStrConvSzStrAndDestruct(pStrB, &pszTAG, 1); + cstrFinalize(pStrB); + cstrConvSzStrAndDestruct(pStrB, &pszTAG, 1); if(pszTAG == NULL) { /* rger, 2005-11-10: no TAG found - this implies that what * we have considered to be the HOSTNAME is most probably the @@ -1611,7 +1610,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the hostname. This situation is the standard case with * stock BSD syslogd. */ - dbgprintf("No TAG in message, assuming that HOSTNAME is missing.\n"); + DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n"); moveHOSTNAMEtoTAG(pMsg); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } else { /* we have a TAG, so we can happily set it ;) */ @@ -1627,7 +1626,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) */ if(!(flags & INTERNAL_MSG)) { - dbgprintf("HOSTNAME and TAG not parsed by user configuraton.\n"); + DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } } @@ -1688,7 +1687,7 @@ logmsg(msg_t *pMsg, int flags) assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); msg = (char*) pMsg->pszUxTradMsg; - dbgprintf("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); + DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have * a traditional syslog message or one formatted according to syslog-protocol. @@ -1696,14 +1695,14 @@ logmsg(msg_t *pMsg, int flags) * -protocol VERSION field for the detection. */ if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); + DBGPRINTF("Message has syslog-protocol format.\n"); setProtocolVersion(pMsg, 1); if(parseRFCSyslogMsg(pMsg, flags) == 1) { msgDestruct(&pMsg); return; } } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); + DBGPRINTF("Message has legacy syslog format.\n"); setProtocolVersion(pMsg, 0); if(parseLegacySyslogMsg(pMsg, flags) == 1) { msgDestruct(&pMsg); @@ -1753,7 +1752,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) * in an acceptable way. -- rgerhards, 2008-09-16 */ if (pAction->f_prevcount && time(NULL) >= REPEATTIME(pAction)) { - dbgprintf("flush %s: repeated %d times, %d sec.\n", + DBGPRINTF("flush %s: repeated %d times, %d sec.\n", module.GetStateName(pAction->pMod), pAction->f_prevcount, repeatinterval[pAction->f_repeatcount]); actionWriteToAction(pAction); @@ -1789,9 +1788,9 @@ static void debug_switch() if(debugging_on == 0) { debugging_on = 1; - dbgprintf("Switching debugging_on to true\n"); + DBGPRINTF("Switching debugging_on to true\n"); } else { - dbgprintf("Switching debugging_on to false\n"); + DBGPRINTF("Switching debugging_on to false\n"); debugging_on = 0; } @@ -1935,7 +1934,7 @@ void legacyOptsParseTCP(char ch, char *arg) * a minimal delay, but it is much cleaner than the approach of doing everything * inside the signal handler. * rgerhards, 2005-10-26 - * Note: we do not call dbgprintf() as this may cause us to block in case something + * Note: we do not call DBGPRINTF() as this may cause us to block in case something * with the threading is wrong. */ static void doDie(int sig) @@ -1983,7 +1982,7 @@ die(int sig) { char buf[256]; - dbgprintf("exiting on signal %d\n", sig); + DBGPRINTF("exiting on signal %d\n", sig); /* IMPORTANT: we should close the inputs first, and THEN send our termination * message. If we do it the other way around, logmsgInternal() may block on @@ -1998,7 +1997,7 @@ die(int sig) */ /* close the inputs */ - dbgprintf("Terminating input threads...\n"); + DBGPRINTF("Terminating input threads...\n"); thrdTerminateAll(); /* and THEN send the termination log message (see long comment above) */ @@ -2012,17 +2011,17 @@ die(int sig) } /* drain queue (if configured so) and stop main queue worker thread pool */ - dbgprintf("Terminating main queue...\n"); + DBGPRINTF("Terminating main queue...\n"); qqueueDestruct(&pMsgQueue); pMsgQueue = NULL; /* Free ressources and close connections. This includes flushing any remaining * repeated msgs. */ - dbgprintf("Terminating outputs...\n"); + DBGPRINTF("Terminating outputs...\n"); freeSelectors(); - dbgprintf("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); + DBGPRINTF("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); /* rger 2005-02-22 * now clean up the in-memory structures. OK, the OS * would also take care of that, but if we do it @@ -2060,7 +2059,7 @@ die(int sig) */ module.UnloadAndDestructAll(eMOD_LINK_ALL); - dbgprintf("Clean shutdown completed, bye\n"); + DBGPRINTF("Clean shutdown completed, bye\n"); /* dbgClassExit MUST be the last one, because it de-inits the debug system */ dbgClassExit(); @@ -2117,7 +2116,7 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) iFiles, errStr, (long) maxFiles.rlim_max); ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); } - dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); + DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); finalize_it: RETiRet; @@ -2128,7 +2127,7 @@ finalize_it: static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { umask(iUmask); - dbgprintf("umask set to 0%3.3o.\n", iUmask); + DBGPRINTF("umask set to 0%3.3o.\n", iUmask); return RS_RET_OK; } @@ -2213,7 +2212,7 @@ static void freeSelectors(void) selector_t *fPrev; if(Files != NULL) { - dbgprintf("Freeing log structures.\n"); + DBGPRINTF("Freeing log structures.\n"); for(f = Files ; f != NULL ; f = f->f_next) { llExecFunc(&f->llActList, freeSelectorsActions, NULL); @@ -2420,7 +2419,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) { DEFiRet; iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); + DBGPRINTF("\n"); RETiRet; } @@ -2437,43 +2436,44 @@ static void dbgPrintInitInfo(void) int iSelNbr = 1; int i; - dbgprintf("\nActive selectors:\n"); + DBGPRINTF("\nActive selectors:\n"); for (f = Files; f != NULL ; f = f->f_next) { - dbgprintf("Selector %d:\n", iSelNbr++); + DBGPRINTF("Selector %d:\n", iSelNbr++); if(f->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); + DBGPRINTF("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); if(f->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", + DBGPRINTF("hostname: %s '%s'\n", f->eHostnameCmpMode == HN_COMP_MATCH ? "only" : "allbut", rsCStrGetSzStrNoNULL(f->pCSHostnameComp)); if(f->f_filter_type == FILTER_PRI) { for (i = 0; i <= LOG_NFACILITIES; i++) - if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", f->f_filterData.f_pmask[i]); + if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) { + DBGPRINTF(" X "); + } else { + DBGPRINTF("%2X ", f->f_filterData.f_pmask[i]); + } } else if(f->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); + DBGPRINTF("EXPRESSION-BASED Filter: can currently not be displayed"); } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", + DBGPRINTF("PROPERTY-BASED Filter:\n"); + DBGPRINTF("\tProperty.: '%s'\n", rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName)); - dbgprintf("\tOperation: "); + DBGPRINTF("\tOperation: "); if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", + DBGPRINTF("NOT "); + DBGPRINTF("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); + DBGPRINTF("\tValue....: '%s'\n", rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue)); - dbgprintf("\tAction...: "); + DBGPRINTF("\tAction...: "); } - dbgprintf("\nActions:\n"); + DBGPRINTF("\nActions:\n"); llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - dbgprintf("\n"); + DBGPRINTF("\n"); } - dbgprintf("\n"); + DBGPRINTF("\n"); if(bDebugPrintTemplateList) tplPrintList(); if(bDebugPrintModuleList) @@ -2483,24 +2483,24 @@ static void dbgPrintInitInfo(void) if(bDebugPrintCfSysLineHandlerList) dbgPrintCfSysLineHandlers(); - dbgprintf("Messages with malicious PTR DNS Records are %sdropped.\n", + DBGPRINTF("Messages with malicious PTR DNS Records are %sdropped.\n", glbl.GetDropMalPTRMsgs() ? "" : "not "); - dbgprintf("Control characters are %sreplaced upon reception.\n", + DBGPRINTF("Control characters are %sreplaced upon reception.\n", bEscapeCCOnRcv? "" : "not "); if(bEscapeCCOnRcv) - dbgprintf("Control character escape sequence prefix is '%c'.\n", + DBGPRINTF("Control character escape sequence prefix is '%c'.\n", cCCEscapeChar); - dbgprintf("Main queue size %d messages.\n", iMainMsgQueueSize); - dbgprintf("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", + DBGPRINTF("Main queue size %d messages.\n", iMainMsgQueueSize); + DBGPRINTF("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", iMainMsgQueueNumWorkers, iMainMsgQtoWrkShutdown, iMainMsgQPersistUpdCnt); - dbgprintf("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", + DBGPRINTF("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", iMainMsgQtoQShutdown, iMainMsgQtoActShutdown, iMainMsgQtoEnq); - dbgprintf("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", + DBGPRINTF("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", iMainMsgQHighWtrMark, iMainMsgQLowWtrMark, iMainMsgQDiscardMark, iMainMsgQDiscardSeverity); - dbgprintf("Main queue save on shutdown %d, max disk space allowed %lld\n", + DBGPRINTF("Main queue save on shutdown %d, max disk space allowed %lld\n", bMainMsgQSaveOnShutdown, iMainMsgQueMaxDiskSpace); /* TODO: add iActionRetryCount = 0; @@ -2511,7 +2511,7 @@ static void dbgPrintInitInfo(void) setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ - dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); + DBGPRINTF("Work Directory: '%s'.\n", glbl.GetWorkDir()); } @@ -2534,7 +2534,7 @@ startInputModules(void) /* activate here */ thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun); } else { - dbgprintf("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); + DBGPRINTF("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); } pMod = module.GetNxtType(pMod, eMOD_IN); } @@ -2567,11 +2567,11 @@ init(void) pDfltProgNameCmp = NULL; eDfltHostnameCmpMode = HN_NO_COMP; - dbgprintf("rsyslog %s - called init()\n", VERSION); + DBGPRINTF("rsyslog %s - called init()\n", VERSION); /* delete the message queue, which also flushes all messages left over */ if(pMsgQueue != NULL) { - dbgprintf("deleting main message queue\n"); + DBGPRINTF("deleting main message queue\n"); qqueueDestruct(&pMsgQueue); /* delete pThis here! */ pMsgQueue = NULL; } @@ -2582,10 +2582,10 @@ init(void) freeSelectors(); /* Unload all non-static modules */ - dbgprintf("Unloading non-static modules.\n"); + DBGPRINTF("Unloading non-static modules.\n"); module.UnloadAndDestructAll(eMOD_LINK_DYNAMIC_LOADED); - dbgprintf("Clearing templates.\n"); + DBGPRINTF("Clearing templates.\n"); tplDeleteNew(); /* re-setting values to defaults (where applicable) */ @@ -2635,7 +2635,7 @@ init(void) snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); conf.cfline((uchar*)cbuf, &f); } else { - dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); + DBGPRINTF("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } selectorAddList(f); } @@ -2741,7 +2741,7 @@ init(void) } bHaveMainQueue = (MainMsgQueType == QUEUETYPE_DIRECT) ? 0 : 1; - dbgprintf("Main processing queue is initialized and running\n"); + DBGPRINTF("Main processing queue is initialized and running\n"); /* the output part and the queue is now ready to run. So it is a good time * to start the inputs. Please note that the net code above should be @@ -2768,7 +2768,7 @@ init(void) sigAct.sa_handler = sighup_handler; sigaction(SIGHUP, &sigAct, NULL); - dbgprintf(" (re)started.\n"); + DBGPRINTF(" (re)started.\n"); finalize_it: RETiRet; @@ -2799,7 +2799,7 @@ selectorAddList(selector_t *f) selectorDestruct(f); } else { /* successfully created an entry */ - dbgprintf("selector line successfully processed\n"); + DBGPRINTF("selector line successfully processed\n"); /* TODO: we should use the linked list class for the selector list, else we need to add globals * ... well nextp could be added temporarily... * Thanks to varmojfekoj for having the idea to just use "Files" to make this @@ -2831,16 +2831,16 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz if (!strcasecmp((char *) pszType, "fixedarray")) { MainMsgQueType = QUEUETYPE_FIXED_ARRAY; - dbgprintf("main message queue type set to FIXED_ARRAY\n"); + DBGPRINTF("main message queue type set to FIXED_ARRAY\n"); } else if (!strcasecmp((char *) pszType, "linkedlist")) { MainMsgQueType = QUEUETYPE_LINKEDLIST; - dbgprintf("main message queue type set to LINKEDLIST\n"); + DBGPRINTF("main message queue type set to LINKEDLIST\n"); } else if (!strcasecmp((char *) pszType, "disk")) { MainMsgQueType = QUEUETYPE_DISK; - dbgprintf("main message queue type set to DISK\n"); + DBGPRINTF("main message queue type set to DISK\n"); } else if (!strcasecmp((char *) pszType, "direct")) { MainMsgQueType = QUEUETYPE_DIRECT; - dbgprintf("main message queue type set to DIRECT (no queueing at all)\n"); + DBGPRINTF("main message queue type set to DIRECT (no queueing at all)\n"); } else { errmsg.LogError(0, RS_RET_INVALID_PARAMS, "unknown mainmessagequeuetype parameter: %s", (char *) pszType); iRet = RS_RET_INVALID_PARAMS; @@ -3219,7 +3219,7 @@ static rsRetVal mainThread() CHKiRet(init()); if(Debug && debugging_on) { - dbgprintf("Debugging enabled, SIGUSR1 to turn off debugging.\n"); + DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n"); } /* Send a signal to the parent so it can terminate. */ @@ -3251,7 +3251,7 @@ static rsRetVal mainThread() * do the init() and then restart things. * rgerhards, 2005-10-24 */ - dbgprintf("initialization completed, transitioning to regular run mode\n"); + DBGPRINTF("initialization completed, transitioning to regular run mode\n"); mainloop(); @@ -3458,7 +3458,7 @@ doGlblProcessInit(void) if( !(Debug || NoFork) ) { - dbgprintf("Checking pidfile.\n"); + DBGPRINTF("Checking pidfile.\n"); if (!check_pid(PidFile)) { memset(&sigAct, 0, sizeof (sigAct)); @@ -3494,7 +3494,7 @@ doGlblProcessInit(void) } /* tuck my process id away */ - dbgprintf("Writing pidfile %s.\n", PidFile); + DBGPRINTF("Writing pidfile %s.\n", PidFile); if (!check_pid(PidFile)) { if (!write_pid(PidFile)) @@ -3640,7 +3640,7 @@ int realMain(int argc, char **argv) if ((argc -= optind)) usage(); - dbgprintf("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", + DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", VERSION, iCompatibilityMode, glblModPath == NULL ? "" : (char*)glblModPath); /* we are done with the initial option parsing and processing. Now we init the system. */ @@ -3722,7 +3722,7 @@ int realMain(int argc, char **argv) /* END core initializations - we now come back to carrying out command line options*/ while((iRet = bufOptRemove(&ch, &arg)) == RS_RET_OK) { - dbgprintf("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); + DBGPRINTF("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); switch((char)ch) { case '4': glbl.SetDefPFFamily(PF_INET); -- cgit From 74b2b24f508be90d20961304d5e3cce648f3eb7c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 12:31:27 +0200 Subject: removed long-obsoleted property UxTradMsg ... as well as some cleanup (some commented-out code is left to support UxTradMsg again is someone really complains ;)). --- ChangeLog | 1 + doc/property_replacer.html | 4 -- plugins/imfile/imfile.c | 1 - plugins/imklog/imklog.c | 2 - runtime/msg.c | 100 ++++++++++++++++++++++----------------------- runtime/msg.h | 9 ++-- runtime/parser.c | 2 +- tools/syslogd.c | 23 ++++------- 8 files changed, 65 insertions(+), 77 deletions(-) diff --git a/ChangeLog b/ChangeLog index f9e3e32b..beed5663 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- removed long-obsoleted property UxTradMsg - added a generic network stream server (in addition to rather specific syslog tcp server) - added ability for the UDP output action to rebind its send socket after diff --git a/doc/property_replacer.html b/doc/property_replacer.html index a6e9b518..7b604ea0 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -30,10 +30,6 @@ Currently supported are:

      socket. Should be useful for debugging. -uxtradmsg -will disappear soon - do NOT use! - - hostname hostname from the message diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index e3882ce5..927cb82e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -96,7 +96,6 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); - MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 45933436..420ebbf1 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -97,8 +97,6 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsg(pMsg, (char*)msg); - MsgSetUxTradMsg(pMsg, (char*)msg); - MsgSetRawMsg(pMsg, (char*)msg); MsgSetMSG(pMsg, (char*)msg); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); diff --git a/runtime/msg.c b/runtime/msg.c index 55cc48b8..8122383a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -343,7 +343,6 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - free(pThis->pszUxTradMsg); free(pThis->pszRawMsg); free(pThis->pszTAG); free(pThis->pszHOSTNAME); @@ -437,6 +436,10 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + pNew->offAfterPRI = pOld->offAfterPRI; + */ memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); @@ -445,7 +448,6 @@ msg_t* MsgDup(msg_t* pOld) tmpCOPYSZ(FacilityStr); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); - tmpCOPYSZ(UxTradMsg); tmpCOPYSZ(TAG); tmpCOPYSZ(HOSTNAME); tmpCOPYSZ(RcvFrom); @@ -494,10 +496,13 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + objSerializeSCALAR(pStrm, offsAfterPRI, SHORT); + */ objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszMSG, PSZ); - objSerializePTR(pStrm, pszUxTradMsg, PSZ); objSerializePTR(pStrm, pszTAG, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); objSerializePTR(pStrm, pszInputName, PSZ); @@ -701,16 +706,17 @@ static char *getRawMsg(msg_t *pM) return (char*)pM->pszRawMsg; } + +/* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 char *getUxTradMsg(msg_t *pM) { if(pM == NULL) return ""; else - if(pM->pszUxTradMsg == NULL) - return ""; - else - return (char*)pM->pszUxTradMsg; + return (char*)pM->pszRawMsg + pM->offAfterPRI; } +*/ char *getMSG(msg_t *pM) { @@ -724,21 +730,30 @@ char *getMSG(msg_t *pM) } +/* Get PRI value as integer */ +static int getPRIi(msg_t *pM) +{ + assert(pM != NULL); + return (pM->iFacility << 3) + (pM->iSeverity); +} + + /* Get PRI value in text form */ static inline char *getPRI(msg_t *pM) { if(pM == NULL) return ""; - return (char*)pM->bufPRI; -} - + /* there are some cases where bufPRI may not contain a valid string, + * and then we need to build it. + */ + MsgLock(pM); + if(pM->bufPRI[0] == '\0') { + snprintf((char*)pM->bufPRI, sizeof(pM->bufPRI), "<%d>", getPRIi(pM)); + } + MsgUnlock(pM); -/* Get PRI value as integer */ -static int getPRIi(msg_t *pM) -{ - assert(pM != NULL); - return (pM->iFacility << 3) + (pM->iSeverity); + return (char*)pM->bufPRI; } @@ -1017,6 +1032,17 @@ MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl) RETiRet; } +/* set offset after which PRI in raw msg starts + * rgerhards, 2009-06-16 + */ +rsRetVal +MsgSetAfterPRIOffs(msg_t *pMsg, short offs) +{ + assert(pMsg != NULL); + pMsg->offAfterPRI = offs; + return RS_RET_OK; +} + /* rgerhards 2004-11-24: set APP-NAME in msg object * TODO: revisit msg locking code! @@ -1520,40 +1546,6 @@ void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) } -/* Set the UxTradMsg to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetUxTradMsg(). - * rgerhards 2004-11-19 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static void MsgAssignUxTradMsg(msg_t *pMsg, char *pBuf) -{ - assert(pMsg != NULL); - assert(pBuf != NULL); - pMsg->iLenUxTradMsg = strlen(pBuf); - pMsg->pszUxTradMsg = pBuf; -} -#endif - - -/* rgerhards 2004-11-17: set the traditional Unix message in msg object - */ -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg) -{ - assert(pMsg != NULL); - assert(pszUxTradMsg != NULL); - pMsg->iLenUxTradMsg = strlen(pszUxTradMsg); - if(pMsg->pszUxTradMsg != NULL) - free(pMsg->pszUxTradMsg); - if((pMsg->pszUxTradMsg = malloc(pMsg->iLenUxTradMsg + 1)) != NULL) - memcpy(pMsg->pszUxTradMsg, pszUxTradMsg, pMsg->iLenUxTradMsg + 1); - else - dbgprintf("Could not allocate memory for pszUxTradMsg buffer."); - - return(0); -} - - /* rgerhards 2004-11-09: set MSG in msg object */ void MsgSetMSG(msg_t *pMsg, char* pszMSG) @@ -1732,8 +1724,11 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = getMSG(pMsg); } else if(!strcmp((char*) pName, "rawmsg")) { pRes = getRawMsg(pMsg); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 } else if(!strcmp((char*) pName, "uxtradmsg")) { pRes = getUxTradMsg(pMsg); + */ } else if(!strcmp((char*) pName, "inputname")) { pRes = (char*) getInputName(pMsg); } else if(!strcmp((char*) pName, "fromhost")) { @@ -2457,10 +2452,15 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) pThis->msgFlags = pProp->val.num; } else if(isProp("pszRawMsg")) { MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(isProp("offAfterPRI")) { + pThis->offAfterPRI = pProp->val.num; + */ } else if(isProp("pszMSG")) { MsgSetMSG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszUxTradMsg")) { - MsgSetUxTradMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { diff --git a/runtime/msg.h b/runtime/msg.h index 20360641..fe9f87fa 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,7 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; -short bDoLock; /* use the mutex? */ + bool bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ @@ -73,13 +73,13 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - //uchar *pszPRI; /* the PRI as a string */ - uchar bufPRI[5]; + uchar bufPRI[5]; /* PRI as string */ int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we * need to preserve cryptographic verifiers. */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ int iLenRawMsg; /* length of raw message */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ @@ -158,7 +158,7 @@ void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); +rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); @@ -183,6 +183,7 @@ uchar *getRcvFrom(msg_t *pM); #if 0 char *getUxTradMsg(msg_t *pM); +int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); #endif /* The MsgPrepareEnqueue() function is a macro for performance reasons. diff --git a/runtime/parser.c b/runtime/parser.c index 13fb51ec..0b45bfd5 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -314,7 +314,7 @@ rsRetVal parseMsg(msg_t *pMsg) } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - MsgSetUxTradMsg(pMsg, (char*) msg); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); if(pMsg->bParseHOSTNAME == 0) MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); diff --git a/tools/syslogd.c b/tools/syslogd.c index c224087d..2f06d273 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -670,16 +670,9 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) MsgSetHOSTNAME(pMsg, hname); MsgSetRcvFrom(pMsg, hname); + MsgSetAfterPRIOffs(pMsg, p - msg); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); - /* rgerhards 2004-11-19: well, well... we've now seen that we - * have the "hostname problem" also with the traditional Unix - * message. As we like to emulate it, we need to add the hostname - * to it. - */ - if(MsgSetUxTradMsg(pMsg, (char*)p) != 0) - ABORT_FINALIZE(RS_RET_ERR); - logmsg(pMsg, flags); finalize_it: @@ -948,7 +941,6 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) CHKiRet(msgConstruct(&pMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); - MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); @@ -1370,8 +1362,8 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ /* do a sanity check on the version and eat it */ assert(p2parse[0] == '1' && p2parse[1] == ' '); @@ -1471,8 +1463,8 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ /* Check to see if msg contains a timestamp. We start by assuming * that the message timestamp is the time of reciption (which we @@ -1685,8 +1677,9 @@ logmsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - msg = (char*) pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + + msg = (char*) pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have -- cgit From a48918cc2783f4351b246c97cd9669abb9d0de47 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 10:09:57 +0200 Subject: minor: prepped ChangeLog for next beta --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 59d9e194..ff1baba9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +Version 4.3.2 [beta] (rgerhards), 2009-06-24 - removed long-obsoleted property UxTradMsg - added a generic network stream server (in addition to rather specific syslog tcp server) -- cgit From 675d46f5b59f64e378968baa5e0dec6810090287 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 10:29:57 +0200 Subject: cleanup (removed left-over from backport) --- tests/Makefile.am | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index a95139f2..88c3137e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -70,8 +70,6 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ - daqueue-persist.sh \ - daqueue-persist-drvr.sh \ testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ -- cgit