diff options
40 files changed, 999 insertions, 78 deletions
@@ -151,13 +151,41 @@ Version 5.9.0 [V5-DEVEL] (rgerhards), 2011-03-?? - enhanced omhdfs to support batching mode. This permits to increase performance, as we now call the HDFS API with much larger message sizes and far more infrequently ---------------------------------------------------------------------------- -Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-?? +- bugfix: failover did not work correctly if repeated msg reduction was on + affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 +--------------------------------------------------------------------------- +Version 5.8.0 [V5-stable] (rgerhards), 2011-04-?? +- bugfix: DA queue was never shutdown once it was started + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241 +--------------------------------------------------------------------------- +Version 5.7.10 [V5-BETA] (rgerhards), 2011-03-29 +- bugfix: ompgsql did not work properly with ANSI SQL strings + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229 +- bugfix: rsyslog did not build with --disable-regexp configure option + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243 +- bugfix: PRI was invalid on Solaris for message from local log socket +- enhance: added $BOM system property to ease writing byte order masks +- bugfix: RFC5424 parser confused by empty structured data + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237 +- bugfix: error return from strgen caused abort, now causes action to be + ignored (just like a failed filter) +- new sample plugin for a strgen to generate sql statement consumable + by a database plugin +- bugfix: strgen could not be used together with database outputs + because the sql/stdsql option could not be specified. This has been + solved by permitting the strgen to include the opton inside its name. + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195 +--------------------------------------------------------------------------- +Version 5.7.9 [V5-BETA] (rgerhards), 2011-03-16 - improved testbench among others, life tests for ommysql (against a test database) have been added, valgrind-based testing enhanced, ... - enhance: fallback *at runtime* to epoll_create if epoll_create1 is not available. Thanks to Michael Biebl for analysis and patch! +- bugfix: failover did not work correctly if repeated msg reduction was on + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 + affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on - bugfix: minor memory leak in omlibdbi (< 1k per instance and run) - bugfix: (regression) omhdfs did no longer compile - bugfix: omlibdbi did not use password from rsyslog.conf @@ -283,6 +311,9 @@ Version 5.7.0 [V5-DEVEL] (rgerhards), 2010-09-16 * sd-systemd API added as part of rsyslog runtime library --------------------------------------------------------------------------- Version 5.6.5 [V5-STABLE] (rgerhards), 2011-03-?? +- bugfix: failover did not work correctly if repeated msg reduction was on + affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on + closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 - bugfix: omlibdbi did not use password from rsyslog.con closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203 - bugfix(kind of): tell users that config graph can currently not be diff --git a/Makefile.am b/Makefile.am index 1a335451..8c00e3da 100644 --- a/Makefile.am +++ b/Makefile.am @@ -112,6 +112,10 @@ if ENABLE_CUST1 SUBDIRS += plugins/cust1 endif +if ENABLE_SMCUSTBINDCDR +SUBDIRS += plugins/sm_cust_bindcdr +endif + if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif @@ -4,7 +4,44 @@ * * File begun on 2007-08-06 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. + * Some notes on processing (this hopefully makes it easier to find + * the right code in question): For performance reasons, this module + * uses different methods of message submission based on the user-selected + * configuration. This code is similar, but can not be abstracted because + * of the performanse-affecting differences in it. As such, it is often + * necessary to triple-check that everything works well in *all* modes. + * The different modes (and calling sequence) are: + * + * if set iExecEveryNthOccur > 1 || f_ReduceRepeated || iSecsExecOnceInterval + * - doSubmitToActionQComplexBatch + * - helperSubmitToActionQComplexBatch + * - doActionCallAction + * handles duplicate message processing, but in essence calls + * - actionWriteToAction + * - qqueueEnqObj + * (now queue engine processing) + * if(pThis->bWriteAllMarkMsgs == FALSE) - this is the DEFAULT + * - doSubmitToActionQNotAllMarkBatch + * - doSubmitToActionQBatch (and from here like in the else case below!) + * else + * - doSubmitToActionQBatch + * - doSubmitToActionQ + * - qqueueEnqObj + * (now queue engine processing) + * + * Note that bWriteAllMakrMsgs on or off creates almost the same processing. + * The difference ist that if WriteAllMarkMsgs is not set, we need to + * preprocess the batch and drop mark messages which are not yet due for + * writing. + * + * After dequeue, processing is as follows: + * - processBatchMain + * - processAction + * - submitBatch + * - tryDoAction + * - + * + * Copyright 2007-2011 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -905,7 +942,7 @@ tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem) if(*(pBatch->pbShutdownImmediate)) ABORT_FINALIZE(RS_RET_FORCE_TERM); if( pBatch->pElem[i].bFilterOK - && pBatch->pElem[i].state != BATCH_STATE_DISC + && pBatch->pElem[i].state != BATCH_STATE_DISC//) { && ((pAction->bExecWhenPrevSusp == 0) || pBatch->pElem[i].bPrevWasSuspended) ) { pMsg = (msg_t*) pBatch->pElem[i].pUsrp; localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams, @@ -1040,7 +1077,8 @@ prepareBatch(action_t *pAction, batch_t *pBatch) pElem = &(pBatch->pElem[i]); if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) { pElem->state = BATCH_STATE_RDY; - prepareDoActionParams(pAction, pElem); + if(prepareDoActionParams(pAction, pElem) != RS_RET_OK) + pElem->bFilterOK = FALSE; } } RETiRet; @@ -1170,11 +1208,33 @@ static rsRetVal setActionQueType(void __attribute__((unused)) *pVal, uchar *pszT } +/* This submits the message to the action queue in case we do NOT need to handle repeat + * message processing. That case permits us to gain lots of freedom during processing + * and thus speed. This is also utilized to submit messages in complex case once + * the complex logic has been applied ;) + * rgerhards, 2010-06-08 + */ +static inline rsRetVal +doSubmitToActionQ(action_t *pAction, msg_t *pMsg) +{ + DEFiRet; + + if(pAction->pQueue->qType == QUEUETYPE_DIRECT) + iRet = qqueueEnqObjDirect(pAction->pQueue, (void*) MsgAddRef(pMsg)); + else + iRet = qqueueEnqObj(pAction->pQueue, pMsg->flowCtlType, (void*) MsgAddRef(pMsg)); + + RETiRet; +} + + /* This function builds up a batch of messages to be (later) * submitted to the action queue. + * Note: this function is also called from syslogd itself as part of its + * flush processing. If so, pBatch will be NULL and idxBtch undefined. */ rsRetVal -actionWriteToAction(action_t *pAction) +actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch) { msg_t *pMsgSave; /* to save current message pointer, necessary to restore it in case it needs to be updated (e.g. repeated msgs) */ @@ -1245,7 +1305,7 @@ actionWriteToAction(action_t *pAction) pAction->f_pMsg = pMsg; /* use the new msg (pointer will be restored below) */ } - DBGPRINTF("Called action, logging to %s\n", module.GetStateName(pAction->pMod)); + DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod)); /* now check if we need to drop the message because otherwise the action would be too * frequently called. -- rgerhards, 2008-04-08 @@ -1263,14 +1323,43 @@ actionWriteToAction(action_t *pAction) FINALIZE; } - /* we use reception time, not dequeue time - this is considered more appropriate and also faster ;) -- rgerhards, 2008-09-17 */ + /* we use reception time, not dequeue time - this is considered more appropriate and also faster ;) + * rgerhards, 2008-09-17 */ pAction->tLastExec = getActNow(pAction); /* re-init time flags */ pAction->f_time = pAction->f_pMsg->ttGenTime; /* When we reach this point, we have a valid, non-disabled action. * So let's enqueue our message for execution. -- rgerhards, 2007-07-24 */ - iRet = qqueueEnqObj(pAction->pQueue, pAction->f_pMsg->flowCtlType, (void*) MsgAddRef(pAction->f_pMsg)); + if( pBatch != NULL + && (pAction->bExecWhenPrevSusp == 1 && pBatch->pElem[idxBtch].bPrevWasSuspended)) { + /* in that case, we need to create a special batch which reflects the + * suspended state. Otherwise, that information would be dropped inside + * the queue engine. TODO: in later releases (v6?) create a better + * solution than what we do here. However, for v5 this sounds much too + * intrusive. -- rgerhardsm, 2011-03-16 + * (Code is copied over from queue.c and slightly modified) + */ + batch_t singleBatch; + batch_obj_t batchObj; + int i; + memset(&batchObj, 0, sizeof(batch_obj_t)); + memset(&singleBatch, 0, sizeof(batch_t)); + batchObj.state = BATCH_STATE_RDY; + batchObj.pUsrp = (obj_t*) pAction->f_pMsg; + batchObj.bPrevWasSuspended = 1; + batchObj.bFilterOK = 1; + singleBatch.nElem = 1; /* there always is only one in direct mode */ + singleBatch.pElem = &batchObj; + + iRet = qqueueEnqObjDirectBatch(pAction->pQueue, &singleBatch); + + for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) { + free(batchObj.staticActStrings[i]); + } + } else { /* standard case, just submit */ + iRet = doSubmitToActionQ(pAction, pAction->f_pMsg); + } if(iRet == RS_RET_OK) pAction->f_prevcount = 0; /* message processed, so we start a new cycle */ @@ -1298,10 +1387,12 @@ finalize_it: * pthread_cleanup_push() POSIX macro... */ static inline rsRetVal -doActionCallAction(action_t *pAction, msg_t *pMsg) +doActionCallAction(action_t *pAction, batch_t *pBatch, int idxBtch) { + msg_t *pMsg; DEFiRet; + pMsg = (msg_t*)(pBatch->pElem[idxBtch].pUsrp); pAction->tActNow = -1; /* we do not yet know our current time (clear prev. value) */ /* don't output marks to recently written outputs */ @@ -1328,7 +1419,7 @@ doActionCallAction(action_t *pAction, msg_t *pMsg) * isolated messages), but back off so we'll flush less often in the future. */ if(getActNow(pAction) > REPEATTIME(pAction)) { - iRet = actionWriteToAction(pAction); + iRet = actionWriteToAction(pAction, pBatch, idxBtch); BACKOFF(pAction); } } else {/* new message, save it */ @@ -1337,7 +1428,7 @@ doActionCallAction(action_t *pAction, msg_t *pMsg) */ if(pAction->f_pMsg != NULL) { if(pAction->f_prevcount > 0) - actionWriteToAction(pAction); + actionWriteToAction(pAction, pBatch, idxBtch); /* we do not care about iRet above - I think it's right but if we have * some troubles, you know where to look at ;) -- rgerhards, 2007-08-01 */ @@ -1345,33 +1436,21 @@ doActionCallAction(action_t *pAction, msg_t *pMsg) } pAction->f_pMsg = MsgAddRef(pMsg); /* call the output driver */ - iRet = actionWriteToAction(pAction); + iRet = actionWriteToAction(pAction, pBatch, idxBtch); } finalize_it: - RETiRet; -} - -/* This submits the message to the action queue in case we do NOT need to handle repeat - * message processing. That case permits us to gain lots of freedom during processing - * and thus speed. - * rgerhards, 2010-06-08 - */ -static inline rsRetVal -doSubmitToActionQ(action_t *pAction, msg_t *pMsg) -{ - DEFiRet; - - if(pAction->pQueue->qType == QUEUETYPE_DIRECT) - iRet = qqueueEnqObjDirect(pAction->pQueue, (void*) MsgAddRef(pMsg)); - else - iRet = qqueueEnqObj(pAction->pQueue, pMsg->flowCtlType, (void*) MsgAddRef(pMsg)); + /* we need to update the batch to handle failover processing correctly */ + if(iRet == RS_RET_OK) { + pBatch->pElem[idxBtch].bPrevWasSuspended = 0; + } else if(iRet == RS_RET_ACTION_FAILED) { + pBatch->pElem[idxBtch].bPrevWasSuspended = 1; + } RETiRet; } - /* This submits the message to the action queue in case where we need to handle * bWriteAllMarkMessage == FALSE only. Note that we use a non-blocking CAS loop * for the synchronization. Here, we just modify the filter condition to be false when @@ -1488,8 +1567,9 @@ helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch) DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod)); for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { if( pBatch->pElem[i].bFilterOK - && pBatch->pElem[i].state != BATCH_STATE_DISC) { - doActionCallAction(pAction, (msg_t*)(pBatch->pElem[i].pUsrp)); + && pBatch->pElem[i].state != BATCH_STATE_DISC + && ((pAction->bExecWhenPrevSusp == 0) || pBatch->pElem[i].bPrevWasSuspended) ) { + doActionCallAction(pAction, pBatch, i); } } @@ -100,28 +100,11 @@ rsRetVal actionDestruct(action_t *pThis); rsRetVal actionDbgPrint(action_t *pThis); rsRetVal actionSetGlobalResumeInterval(int iNewVal); rsRetVal actionDoAction(action_t *pAction); -rsRetVal actionWriteToAction(action_t *pAction); +rsRetVal actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch); rsRetVal actionCallHUPHdlr(action_t *pAction); rsRetVal actionClassInit(void); rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, int bSuspended); rsRetVal actionNewScope(void); rsRetVal actionRestoreScope(void); -#if 1 -#define actionIsSuspended(pThis) ((pThis)->bSuspended == 1) -#else -/* The function is a debugging aid */ -inline int actionIsSuspended(action_t *pThis) -{ - int i; - ASSERT(pThis != NULL); - i = pThis->bSuspended == 1; - dbgprintf("in IsSuspend(), returns %d\n", i); - return i; -} -#endif - #endif /* #ifndef ACTION_H_INCLUDED */ -/* - * vi:set ai: - */ diff --git a/configure.ac b/configure.ac index d0de97cc..4a2e354a 100644 --- a/configure.ac +++ b/configure.ac @@ -1117,6 +1117,20 @@ AC_ARG_ENABLE(cust1, AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes) +# A custom strgen that also serves as a sample of how to do +# SQL-generating strgen's +AC_ARG_ENABLE(smcustbindcdr, + [AS_HELP_STRING([--enable-smcustbindcdr],[Compiles smcustbindcdr module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_smcustbindcdr="yes" ;; + no) enable_smcustbindcdr="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-smcustbindcdr) ;; + esac], + [enable_smcustbindcdr=no] +) +AM_CONDITIONAL(ENABLE_SMCUSTBINDCDR, test x$enable_smcustbindcdr = xyes) + + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -1235,6 +1249,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omoracle/Makefile \ plugins/omudpspoof/Makefile \ plugins/mmnormalize/Makefile \ + plugins/sm_cust_bindcdr/Makefile \ plugins/cust1/Makefile \ java/Makefile \ tests/Makefile]) @@ -1285,6 +1300,9 @@ echo echo "---{ message modification modules }---" echo " mmnormalize module will be compiled: $enable_mmnormalize" echo +echo "---{ strgen modules }---" +echo " sm_cust_bindcdr module will be compiled: $enable_sm_cust_bindcdr" +echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" echo " libdbi support enabled: $enable_libdbi" diff --git a/doc/property_replacer.html b/doc/property_replacer.html index 4d242a34..cd357f67 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -156,6 +156,12 @@ than messages generated somewhere. </td> </tr> <tr> +<td><b>$bom</b></td> +<td>The UTF-8 encoded Unicode byte-order mask (BOM). This may be useful +in templates for RFC5424 support, when the character set is know to be +Unicode.</td> +</tr> +<tr> <td><b>$now</b></td> <td>The current date stamp in the format YYYY-MM-DD</td> </tr> diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index c0ccfd03..12b3318d 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -205,7 +205,7 @@ doInjectMsg(int iNum) DEFiRet; snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), - "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum); + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum); datetime.getCurrTime(&stTime, &ttGenTime); /* we now create our own message object and submit it to the queue */ @@ -247,6 +247,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); + DBGPRINTF("imdiag: %d messages injected\n", nMsgs); finalize_it: RETiRet; @@ -279,6 +280,7 @@ waitMainQEmpty(tcps_sess_t *pSess) } CHKiRet(sendResponse(pSess, "mainqueue empty\n")); + DBGPRINTF("imdiag: mainqueue empty\n"); finalize_it: RETiRet; @@ -314,6 +316,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); + DBGPRINTF("imdiag: %d messages in main queue\n", iMsgQueueSize); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) { CHKiRet(waitMainQEmpty(pSess)); } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) { diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 69fd244e..df9cc3fe 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -159,8 +159,13 @@ static rsRetVal initPgSQL(instanceData *pData, int bSilent) dbgprintf("host=%s dbname=%s uid=%s\n",pData->f_dbsrv,pData->f_dbname,pData->f_dbuid); + /* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may + * get all sorts of side effects (e.g.: backslash escapes) and warnings + */ + const char *PgConnectionOptions = "-c standard_conforming_strings=on"; + /* Connect to database */ - if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, NULL, NULL, + if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, PgConnectionOptions, NULL, pData->f_dbname, pData->f_dbuid, pData->f_dbpwd)) == NULL) { reportDBError(pData, bSilent); closePgSQL(pData); /* ignore any error we may get */ diff --git a/plugins/sm_cust_bindcdr/Makefile.am b/plugins/sm_cust_bindcdr/Makefile.am new file mode 100644 index 00000000..1f71d499 --- /dev/null +++ b/plugins/sm_cust_bindcdr/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = sm_cust_bindcdr.la + +sm_cust_bindcdr_la_SOURCES = sm_cust_bindcdr.c +sm_cust_bindcdr_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +sm_cust_bindcdr_la_LDFLAGS = -module -avoid-version +sm_cust_bindcdr_la_LIBADD = diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c new file mode 100644 index 00000000..91394772 --- /dev/null +++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c @@ -0,0 +1,382 @@ +/* sm_cust_bindcdr.c + * This is a custom developed plugin to process bind information into + * a specific SQL statement. While the actual processing may be too specific + * to be of general use, this module serves as a template on how this type + * of processing can be done. + * + * Format generated: + * "%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" + * Note that this is the same as smtradfile.c, except that we do have a RFC3339 timestamp. However, + * we have copied over the code from there, it is too simple to go through all the hassle + * of having a single code base. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2011-03-17 by RGerhards + * + * Copyright 2011 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <ctype.h> +#include <string.h> +#include <assert.h> +#include <errno.h> +#include "conf.h" +#include "syslogd-types.h" +#include "cfsysline.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "unicode-helper.h" +#include "errmsg.h" + +MODULE_TYPE_STRGEN +MODULE_TYPE_NOKEEP +STRGEN_NAME("Custom_BindCDR,sql") + +/* internal structures + */ +DEF_SMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +/* list of "allowed" IPs */ +typedef struct allowedip_s { + uchar *pszIP; + struct allowedip_s *next; +} allowedip_t; + +static allowedip_t *root; + + +/* config data */ + +/* check if the provided IP is (already) in the allowed list + */ +static int +isAllowed(uchar *pszIP) +{ + allowedip_t *pallow; + int ret = 0; + + for(pallow = root ; pallow != NULL ; pallow = pallow->next) { + if(!ustrcmp(pallow->pszIP, pszIP)) { + ret = 1; + goto finalize_it; + } + } +finalize_it: return ret; +} + +/* This function is called to add an additional allowed IP. It adds + * the IP to the linked list of them. An error is emitted if the IP + * already exists. + */ +static rsRetVal addAllowedIP(void __attribute__((unused)) *pVal, uchar *pNewVal) +{ + allowedip_t *pNew; + DEFiRet; + + if(isAllowed(pNewVal)) { + errmsg.LogError(0, NO_ERRCODE, "error: allowed IP '%s' already configured " + "duplicate ignored", pNewVal); + ABORT_FINALIZE(RS_RET_ERR); + } + + CHKmalloc(pNew = malloc(sizeof(allowedip_t))); + pNew->pszIP = pNewVal; + pNew->next = root; + root = pNew; + DBGPRINTF("sm_cust_bindcdr: allowed IP '%s' added.\n", pNewVal); + +finalize_it: + if(iRet != RS_RET_OK) { + free(pNewVal); + } + + RETiRet; +} + +/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings + * needed (including their length) and then calculating the actual space required. So when we + * finally copy, we know exactly what we need. So we do at most one alloc. + * An actual message sample for what we intend to parse is (one line): + <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96) + */ +#define SQL_STMT "INSERT INTO CDR(`Date`,`Time`, timeMS, client, view, query, ip) VALUES ('" +#define ADD_SQL_DELIM \ + memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \ + iBuf += sizeof("', '") - 1; +#define SQL_STMT_END "');\n" +BEGINstrgen + int iBuf; + uchar *psz; + uchar szDate[64]; + unsigned lenDate; + uchar szTime[64]; + unsigned lenTime; + uchar szMSec[64]; + unsigned lenMSec; + uchar szClient[64]; + unsigned lenClient; + uchar szView[64]; + unsigned lenView; + uchar szQuery[64]; + unsigned lenQuery; + uchar szIP[64]; + unsigned lenIP; + size_t lenTotal; +CODESTARTstrgen + /* first create an empty statement. This is to be replaced if + * we have better data to fill in. + */ + /* now make sure buffer is large enough */ + if(*pLenBuf < 2) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, 2)); + memcpy(*ppBuf, ";", sizeof(";")); + + /* first obtain all strings and their length (if not fixed) */ + /* Note that there are two date fields present, one in the header + * and one more in the actual message. We use the one from the message + * and parse that our. We check validity based on some fixe fields. In- + * depth verification is probably not worth the effort (CPU time), because + * we do various other checks on the message format below). + */ + psz = getMSG(pMsg); + if(psz[0] == ' ' && psz[3] == '-' && psz[7] == '-') { + memcpy(szDate, psz+8, 4); + szDate[4] = '-'; + if(!strncmp((char*)psz+4, "Jan", 3)) { + szDate[5] = '0'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Feb", 3)) { + szDate[5] = '0'; + szDate[6] = '2'; + } else if(!strncmp((char*)psz+4, "Mar", 3)) { + szDate[5] = '0'; + szDate[6] = '3'; + } else if(!strncmp((char*)psz+4, "Apr", 3)) { + szDate[5] = '0'; + szDate[6] = '4'; + } else if(!strncmp((char*)psz+4, "May", 3)) { + szDate[5] = '0'; + szDate[6] = '5'; + } else if(!strncmp((char*)psz+4, "Jun", 3)) { + szDate[5] = '0'; + szDate[6] = '6'; + } else if(!strncmp((char*)psz+4, "Jul", 3)) { + szDate[5] = '0'; + szDate[6] = '7'; + } else if(!strncmp((char*)psz+4, "Aug", 3)) { + szDate[5] = '0'; + szDate[6] = '8'; + } else if(!strncmp((char*)psz+4, "Sep", 3)) { + szDate[5] = '0'; + szDate[6] = '9'; + } else if(!strncmp((char*)psz+4, "Oct", 3)) { + szDate[5] = '1'; + szDate[6] = '0'; + } else if(!strncmp((char*)psz+4, "Nov", 3)) { + szDate[5] = '1'; + szDate[6] = '1'; + } else if(!strncmp((char*)psz+4, "Dec", 3)) { + szDate[5] = '1'; + szDate[6] = '2'; + } + szDate[7] = '-'; + szDate[8] = psz[1]; + szDate[9] = psz[2]; + szDate[10] = '\0'; + lenDate = 10; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + + /* now time (pull both regular time and ms) */ + if(psz[12] == ' ' && psz[15] == ':' && psz[18] == ':' && psz[21] == '.' && psz[25] == ' ') { + memcpy(szTime, (char*)psz+13, 8); + szTime[9] = '\0'; + lenTime = 8; + memcpy(szMSec, (char*)psz+22, 3); + szMSec[4] = '\0'; + lenMSec = 3; + } else { + dbgprintf("Custom_BindCDR: date part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + + /* "client" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "client "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: client part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("client ") - 1; /* skip "label" */ + for( lenClient = 0 + ; *psz && *psz != '#' && lenClient < sizeof(szClient) - 1 + ; ++lenClient) { + szClient[lenClient] = *psz++; + } + szClient[lenClient] = '\0'; + } + + /* "view" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "view "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: view part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("view ") - 1; /* skip "label" */ + for( lenView = 0 + ; *psz && *psz != ':' && lenView < sizeof(szView) - 1 + ; ++lenView) { + szView[lenView] = *psz++; + } + szView[lenView] = '\0'; + } + + /* "query" - we must extract just the number, and in reverse! */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "query: "); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: query part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("query: ") - 1; /* skip "label" */ + /* first find end-of-strihttp://www.rsyslog.com/doc/omruleset.htmlng to process */ + while(*psz && (isdigit(*psz) || *psz == '.')) { + psz++; + } + /* now shuffle data */ + for( lenQuery = 0 + ; *psz && *psz != ' ' && lenQuery < sizeof(szQuery) - 1 + ; --psz) { + if(isdigit(*psz)) + szQuery[lenQuery++] = *psz; + } + szQuery[lenQuery] = '\0'; + } + + /* "ip" */ + psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + ("); + if(psz == NULL) { + dbgprintf("Custom_BindCDR: ip part in msg missing\n"); + ABORT_FINALIZE(RS_RET_ERR); + } else { + psz += sizeof("IN TXT + (") - 1; /* skip "label" */ + for( lenIP = 0 + ; *psz && *psz != ')' && lenIP < sizeof(szIP) - 1 + ; ++lenIP) { + szIP[lenIP] = *psz++; + } + szIP[lenIP] = '\0'; + } + + + /* --- strings extracted ---- */ + + /* now check if the IP is "allowed", in which case we should not + * insert into the database. + */ + if(isAllowed(szIP)) { + DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + + /* calculate len, constants for spaces and similar fixed strings */ + lenTotal = lenDate + lenTime + lenMSec + lenClient + lenView + lenQuery + + lenIP + 7 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2; + + /* now make sure buffer is large enough */ + if(lenTotal >= *pLenBuf) + CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal)); + + /* and concatenate the resulting string */ + memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1); + iBuf = sizeof(SQL_STMT) - 1; + + memcpy(*ppBuf + iBuf, szDate, lenDate); + iBuf += lenDate; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szTime, lenTime); + iBuf += lenTime; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szMSec, lenMSec); + iBuf += lenMSec; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szClient, lenClient); + iBuf += lenClient; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szView, lenView); + iBuf += lenView; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szQuery, lenQuery); + iBuf += lenQuery; + ADD_SQL_DELIM + + memcpy(*ppBuf + iBuf, szIP, lenIP); + iBuf += lenIP; + + /* end of SQL statement/trailer (NUL is contained in string!) */ + memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END)); + iBuf += sizeof(SQL_STMT_END); + +finalize_it: +ENDstrgen + + +BEGINmodExit + allowedip_t *pallow, *pdel; +CODESTARTmodExit + for(pallow = root ; pallow != NULL ; ) { + pdel = pallow; + pallow = pallow->next; + free(pdel->pszIP); + free(pdel); + } + + objRelease(errmsg, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_SMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + root = NULL; + CHKiRet(omsdRegCFSLineHdlr((uchar *)"sgcustombindcdrallowedip", 0, eCmdHdlrGetWord, + addAllowedIP, NULL, STD_LOADABLE_MODULE_ID, eConfObjGlobal)); + dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION); +ENDmodInit diff --git a/runtime/msg.c b/runtime/msg.c index ec132489..8016068b 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -449,6 +449,8 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) *pPropID = PROP_CEE_ALL_JSON; } else if(!strncmp((char*) pName, "$!", 2)) { *pPropID = PROP_CEE; + } else if(!strcmp((char*) pName, "$bom")) { + *pPropID = PROP_SYS_BOM; } else { *pPropID = PROP_INVALID; iRet = RS_RET_VAR_NOT_FOUND; @@ -534,6 +536,8 @@ uchar *propIDToName(propid_t propID) return UCHAR_CONSTANT("*CEE-based property*"); case PROP_CEE_ALL_JSON: return UCHAR_CONSTANT("$!all-json"); + case PROP_SYS_BOM: + return UCHAR_CONSTANT("$BOM"); default: return UCHAR_CONSTANT("*invalid property id*"); } @@ -2483,6 +2487,11 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, break; case PROP_CEE: getCEEPropVal(pMsg, propName, &pRes, &bufLen, pbMustBeFreed); + case PROP_SYS_BOM: + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (uchar*) "\xEF\xBB\xBF"; + *pbMustBeFreed = 0; break; default: /* there is no point in continuing, we may even otherwise render the diff --git a/runtime/parser.c b/runtime/parser.c index 8776a25e..2e991a9d 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -453,10 +453,10 @@ ParsePRI(msg_t *pMsg) if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) pri = DEFUPRI; } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); } - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); - MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); RETiRet; } diff --git a/runtime/queue.c b/runtime/queue.c index ef6e843b..50ae307c 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1774,9 +1774,13 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) { DEFiRet; +//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis)); if(pThis->bEnqOnly) { iRet = RS_RET_TERMINATE_WHEN_IDLE; } + if(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk) { + iRet = RS_RET_TERMINATE_NOW; + } RETiRet; } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 9d3d0289..23547535 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -139,6 +139,7 @@ typedef uintTiny propid_t; #define PROP_SYS_MYHOSTNAME 158 #define PROP_CEE 200 #define PROP_CEE_ALL_JSON 201 +#define PROP_SYS_BOM 159 /* The error codes below are orginally "borrowed" from @@ -42,10 +42,13 @@ /* static data */ DEFobjCurrIf(obj) DEFobjCurrIf(errmsg) -DEFobjCurrIf(regexp) DEFobjCurrIf(strgen) +#ifdef FEATURE_REGEXP +DEFobjCurrIf(regexp) static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */ +#endif + static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ @@ -551,10 +554,9 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) cstr_t *pStrB; struct templateEntry *pTpe; int iNum; /* to compute numbers */ - rsRetVal iRetLocal; - #ifdef FEATURE_REGEXP /* APR: variables for regex */ + rsRetVal iRetLocal; int longitud; unsigned char *regex_char; unsigned char *regex_end; @@ -865,19 +867,37 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) { uchar *pSrc, *pDst; uchar szMod[2048]; + unsigned lenMod; strgen_t *pStrgen; DEFiRet; pSrc = *ppRestOfConfLine; pDst = szMod; - while(*pSrc && !isspace(*pSrc) && pDst < &(szMod[sizeof(szMod) - 1])) { - *pDst++ = *pSrc++; + lenMod = 0; + while(*pSrc && !isspace(*pSrc) && lenMod < sizeof(szMod) - 1) { + szMod[lenMod] = *pSrc++; + lenMod++; + } - *pDst = '\0'; + szMod[lenMod] = '\0'; *ppRestOfConfLine = pSrc; CHKiRet(strgen.FindStrgen(&pStrgen, szMod)); pTpl->pStrgen = pStrgen->pModule->mod.sm.strgen; - dbgprintf("template bound to strgen '%s'\n", szMod); + DBGPRINTF("template bound to strgen '%s'\n", szMod); + /* check if the name potentially contains some well-known options + * Note: we have opted to let the name contain all options. This sounds + * useful, because the strgen MUST actually implement a specific set + * of options. Doing this via the name looks to the enduser as if the + * regular syntax were used, and it make sure the strgen postively + * acknowledged implementing the option. -- rgerhards, 2011-03-21 + */ + if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) { + pTpl->optFormatForSQL = 2; + DBGPRINTF("strgen suports the stdsql option\n"); + } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) { + pTpl->optFormatForSQL = 1; + DBGPRINTF("strgen suports the sql option\n"); + } finalize_it: RETiRet; @@ -945,8 +965,7 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) /* we simply make the template defunct in this case by setting * its name to a zero-string. We do not free it, as this would * require additional code and causes only a very small memory - * consumption. Memory is freed, however, in normal operation - * and most importantly by HUPing syslogd. + * consumption. */ *pTpl->pszName = '\0'; return NULL; @@ -1076,6 +1095,7 @@ void tplDeleteAll(void) break; case FIELD: /* check if we have a regexp and, if so, delete it */ +#ifdef FEATURE_REGEXP if(pTpeDel->data.field.has_regex != 0) { if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { regexp.regfree(&(pTpeDel->data.field.re)); @@ -1083,6 +1103,7 @@ void tplDeleteAll(void) } if(pTpeDel->data.field.propName != NULL) es_deleteStr(pTpeDel->data.field.propName); +#endif break; } /*dbgprintf("\n");*/ @@ -1131,6 +1152,7 @@ void tplDeleteNew(void) free(pTpeDel->data.constant.pConstant); break; case FIELD: +#ifdef FEATURE_REGEXP /* check if we have a regexp and, if so, delete it */ if(pTpeDel->data.field.has_regex != 0) { if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { @@ -1139,6 +1161,7 @@ void tplDeleteNew(void) } if(pTpeDel->data.field.propName != NULL) es_deleteStr(pTpeDel->data.field.propName); +#endif break; } /*dbgprintf("\n");*/ diff --git a/tests/Makefile.am b/tests/Makefile.am index 1a6c8f52..bde06bcc 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -55,7 +55,11 @@ TESTS += \ discard-rptdmsg.sh \ discard-allmark.sh \ discard.sh \ - queue-persist.sh \ + failover-basic.sh \ + failover-rptd.sh \ + failover-no-rptd.sh \ + failover-no-basic.sh \ + rcvr_fail_restore.sh \ linkedlistqueue.sh endif @@ -63,6 +67,10 @@ if HAVE_VALGRIND TESTS += \ discard-rptdmsg-vg.sh \ discard-allmark-vg.sh \ + failover-basic-vg.sh \ + failover-rptd-vg.sh \ + failover-no-basic-vg.sh \ + failover-no-rptd-vg.sh \ tcp-msgreduc-vg.sh endif @@ -83,7 +91,6 @@ TESTS += \ endif endif - if ENABLE_IMPTCP TESTS += \ manyptcp.sh \ @@ -278,6 +285,18 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ omod-if-array.sh \ discard.sh \ testsuites/discard.conf \ + failover-no-rptd.sh \ + failover-no-rptd-vg.sh \ + testsuites/failover-no-rptd.conf \ + failover-no-basic.sh \ + failover-no-basic-vg.sh \ + testsuites/failover-no-basic.conf \ + failover-rptd.sh \ + failover-rptd-vg.sh \ + testsuites/failover-rptd.conf \ + failover-basic.sh \ + failover-basic-vg.sh \ + testsuites/failover-basic.conf \ discard-rptdmsg.sh \ discard-rptdmsg-vg.sh \ testsuites/discard-rptdmsg.conf \ @@ -287,6 +306,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ diag.sh \ testsuites/diag-common.conf \ testsuites/diag-common2.conf \ + rcvr_fail_restore.sh \ + testsuites/rcvr_fail_restore_rcvr.conf \ + testsuites/rcvr_fail_restore_sender.conf \ daqueue-persist.sh \ daqueue-persist-drvr.sh \ queue-persist.sh \ diff --git a/tests/diag.sh b/tests/diag.sh index b8b55f11..8b0ad573 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -10,7 +10,7 @@ #valgrind="valgrind --tool=helgrind --log-fd=1" #valgrind="valgrind --tool=exp-ptrcheck --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUG="debug nologfuncflow nostdout" #export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason @@ -84,15 +84,27 @@ case $1 in exit 1 fi ;; + 'get-mainqueuesize') # show the current main queue size + if [ "$2" == "2" ] + then + echo getmainmsgqueuesize | ./diagtalker -p13501 + else + echo getmainmsgqueuesize | ./diagtalker + fi + ;; 'wait-queueempty') # wait for main message queue to be empty. $2 is the instance. if [ "$2" == "2" ] then - echo WaitMainQueueEmpty | ./diagtalker + echo WaitMainQueueEmpty | ./diagtalker -p13501 else echo WaitMainQueueEmpty | ./diagtalker fi ;; 'shutdown-when-empty') # shut rsyslogd down when main queue is empty. $2 is the instance. + if [ "$2" == "2" ] + then + echo Shutting down instance 2 + fi $srcdir/diag.sh wait-queueempty $2 kill `cat rsyslog$2.pid` # note: we do not wait for the actual termination! diff --git a/tests/diagtalker.c b/tests/diagtalker.c index 6a721e47..0d728bda 100644 --- a/tests/diagtalker.c +++ b/tests/diagtalker.c @@ -1,7 +1,7 @@ /* A yet very simple tool to talk to imdiag (this replaces the * previous Java implementation in order to get fewer dependencies). * - * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2010,2011 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -126,7 +126,7 @@ doProcessing() len = strlen(line); sendCmd(fd, line, len); waitRsp(fd, line, sizeof(line)); - printf("imdiag: %s", line); + printf("imdiag[%d]: %s", targetPort, line); } } @@ -139,7 +139,7 @@ int main(int argc, char *argv[]) int ret = 0; int opt; - while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) { + while((opt = getopt(argc, argv, "t:p:")) != -1) { switch (opt) { case 't': targetIP = optarg; break; diff --git a/tests/failover-async.sh b/tests/failover-async.sh new file mode 100755 index 00000000..f17bc0ff --- /dev/null +++ b/tests/failover-async.sh @@ -0,0 +1,12 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-async.sh\]: async test for failover functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup failover-async.conf +source $srcdir/diag.sh injectmsg 0 5 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/failover-basic-vg.sh b/tests/failover-basic-vg.sh new file mode 100755 index 00000000..0eb77caa --- /dev/null +++ b/tests/failover-basic-vg.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-basic.sh\]: basic test for failover functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/failover-basic.sh b/tests/failover-basic.sh new file mode 100755 index 00000000..031ea25b --- /dev/null +++ b/tests/failover-basic.sh @@ -0,0 +1,12 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-basic.sh\]: basic test for failover functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup failover-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/failover-no-basic-vg.sh b/tests/failover-no-basic-vg.sh new file mode 100755 index 00000000..266163d4 --- /dev/null +++ b/tests/failover-no-basic-vg.sh @@ -0,0 +1,20 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-no-basic.sh\]: basic test for failover functionality - no failover +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-no-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +# now we need our custom logic to see if the result file is empty +# (what it should be!) +cmp rsyslog.out.log /dev/null +if [ $? -eq 1 ] +then + echo "ERROR, output file not empty" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/failover-no-basic.sh b/tests/failover-no-basic.sh new file mode 100755 index 00000000..6177e10d --- /dev/null +++ b/tests/failover-no-basic.sh @@ -0,0 +1,19 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-no-basic.sh\]: basic test for failover functionality - no failover +source $srcdir/diag.sh init +source $srcdir/diag.sh startup failover-no-basic.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +# now we need our custom logic to see if the result file is empty +# (what it should be!) +cmp rsyslog.out.log /dev/null +if [ $? -eq 1 ] +then + echo "ERROR, output file not empty" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/failover-no-rptd-vg.sh b/tests/failover-no-rptd-vg.sh new file mode 100755 index 00000000..3b77a0a4 --- /dev/null +++ b/tests/failover-no-rptd-vg.sh @@ -0,0 +1,20 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-no-rptd.sh\]: rptd test for failover functionality - no failover +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-no-rptd.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +# now we need our custom logic to see if the result file is empty +# (what it should be!) +cmp rsyslog.out.log /dev/null +if [ $? -eq 1 ] +then + echo "ERROR, output file not empty" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/failover-no-rptd.sh b/tests/failover-no-rptd.sh new file mode 100755 index 00000000..6abeba44 --- /dev/null +++ b/tests/failover-no-rptd.sh @@ -0,0 +1,19 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-no-rptd.sh\]: rptd test for failover functionality - no failover +source $srcdir/diag.sh init +source $srcdir/diag.sh startup failover-no-rptd.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +# now we need our custom logic to see if the result file is empty +# (what it should be!) +cmp rsyslog.out.log /dev/null +if [ $? -eq 1 ] +then + echo "ERROR, output file not empty" + exit 1 +fi +source $srcdir/diag.sh exit diff --git a/tests/failover-rptd-vg.sh b/tests/failover-rptd-vg.sh new file mode 100755 index 00000000..d208003b --- /dev/null +++ b/tests/failover-rptd-vg.sh @@ -0,0 +1,13 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-rptd.sh\]: rptd test for failover functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup-vg failover-rptd.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown-vg +source $srcdir/diag.sh check-exit-vg +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/failover-rptd.sh b/tests/failover-rptd.sh new file mode 100755 index 00000000..8a313e9e --- /dev/null +++ b/tests/failover-rptd.sh @@ -0,0 +1,12 @@ +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[failover-rptd.sh\]: rptd test for failover functionality +source $srcdir/diag.sh init +source $srcdir/diag.sh startup failover-rptd.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh new file mode 100755 index 00000000..79486f10 --- /dev/null +++ b/tests/rcvr_fail_restore.sh @@ -0,0 +1,122 @@ +# Copyright (C) 2011 by Rainer Gerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[rcvr_fail_restore.sh\]: test failed receiver restore case +source $srcdir/diag.sh init +# +# STEP1: start both instances and send 1000 messages. +# Note: receiver is instance 2, sender instance 1. +# +# start up the instances. Note that the envrionment settings can be changed to +# set instance-specific debugging parameters! +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="log2" +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +#export RSYSLOG_DEBUGLOG="log" +#valgrind="valgrind" +source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf +# re-set params so that new instances do not thrash it... +#unset RSYSLOG_DEBUG +#unset RSYSLOG_DEBUGLOG + +# now inject the messages into instance 2. It will connect to instance 1, +# and that instance will record the data. +source $srcdir/diag.sh injectmsg 1 1000 +source $srcdir/diag.sh wait-queueempty +./msleep 1000 # let things settle down a bit + +# +# Step 2: shutdown receiver, then send some more data, which then +# needs to go into the queue. +# + +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +source $srcdir/diag.sh injectmsg 1001 10000 +./msleep 3000 # make sure some retries happen (retry interval is set to 3 second) +source $srcdir/diag.sh get-mainqueuesize +ls -l test-spool + +# +# Step 3: restart receiver, wait that the sender drains its queue +# +#export RSYSLOG_DEBUGLOG="log2" +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +echo waiting for sender to drain queue [may need a short while] +source $srcdir/diag.sh wait-queueempty +ls -l test-spool +OLDFILESIZE=$(stat -c%s test-spool/mainq.00000001) +echo file size to expect is $OLDFILESIZE + + +# +# Step 4: send new data. Queue files are not permitted to grow now +# (but one file continous to exist). +# +source $srcdir/diag.sh injectmsg 11001 10 +source $srcdir/diag.sh wait-queueempty + +# at this point, the queue file shall not have grown. Note +# that we MUST NOT shut down the instance right now, because it +# would clean up the queue files! So we need to do our checks +# first (here!). +ls -l test-spool +NEWFILESIZE=$(stat -c%s test-spool/mainq.00000001) +if [ $NEWFILESIZE != $OLDFILESIZE ] +then + echo file sizes do not match, expected $OLDFILESIZE, actual $NEWFILESIZE + echo this means that data has been written to the queue file where it + echo no longer should be written. + # abort will happen below, because we must ensure proper system shutdown + # HOWEVER, during actual testing it may be useful to do an exit here (so + # that e.g. the debug log is pointed right at the correct spot). + # exit 1 +fi + +# +# We now do an extra test (so this is two in one ;)) to see if the DA +# queue can be reactivated after its initial shutdown. In essence, we +# redo steps 2 and 3. +# +# Step 5: stop receiver again, then send some more data, which then +# needs to go into the queue. +# +echo "*** done primary test *** now checking if DA can be restarted" +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +source $srcdir/diag.sh injectmsg 11011 10000 +sleep 1 # we need to wait, otherwise we may be so fast that the receiver +# comes up before we have finally suspended the action +source $srcdir/diag.sh get-mainqueuesize +ls -l test-spool + +# +# Step 6: restart receiver, wait that the sender drains its queue +# +source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2 +echo waiting for sender to drain queue [may need a short while] +source $srcdir/diag.sh wait-queueempty +ls -l test-spool + +# +# Queue file size checks done. Now it is time to terminate the system +# and see if everything could be received (the usual check, done here +# for completeness, more or less as a bonus). +# +source $srcdir/diag.sh shutdown-when-empty +source $srcdir/diag.sh wait-shutdown + +# now it is time to stop the receiver as well +source $srcdir/diag.sh shutdown-when-empty 2 +source $srcdir/diag.sh wait-shutdown 2 + +# now abort test if we need to (due to filesize predicate) +if [ $NEWFILESIZE != $OLDFILESIZE ] +then + exit 1 +fi +# do the final check +source $srcdir/diag.sh seq-check 1 21010 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf index 9e9e28fe..d76e2d15 100644 --- a/tests/testsuites/diag-common.conf +++ b/tests/testsuites/diag-common.conf @@ -13,4 +13,7 @@ $IMDiagServerRun 13500 $template startupfile,"rsyslogd.started" # trick to use relative path names! :syslogtag, contains, "rsyslogd" ?startupfile -$ErrorMessagesToStderr off +# I have disabled the directive below, so that we see errors in testcase +# creation. I am not sure why it was present in the first place, so for +# now I just leave it commented out -- rgerhards, 2011-03-30 +#$ErrorMessagesToStderr off diff --git a/tests/testsuites/failover-async.conf b/tests/testsuites/failover-async.conf new file mode 100644 index 00000000..76445de3 --- /dev/null +++ b/tests/testsuites/failover-async.conf @@ -0,0 +1,9 @@ +# see the equally-named .sh file for details +$IncludeConfig diag-common.conf + +$template outfmt,"%msg:F,58:2%\n" +# note: the target server shall not be available! + +$ActionQueueType LinkedList +:msg, contains, "msgnum:" @@127.0.0.1:13514 +& ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/failover-basic.conf b/tests/testsuites/failover-basic.conf new file mode 100644 index 00000000..a858769c --- /dev/null +++ b/tests/testsuites/failover-basic.conf @@ -0,0 +1,8 @@ +# see the equally-named .sh file for details +$IncludeConfig diag-common.conf + +$template outfmt,"%msg:F,58:2%\n" +# note: the target server shall not be available! +:msg, contains, "msgnum:" @@127.0.0.1:13514 +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/failover-no-basic.conf b/tests/testsuites/failover-no-basic.conf new file mode 100644 index 00000000..b40ef7d7 --- /dev/null +++ b/tests/testsuites/failover-no-basic.conf @@ -0,0 +1,9 @@ +# see the equally-named .sh file for details +$IncludeConfig diag-common.conf + +$RepeatedMsgReduction off + +# second action should never execute +:msg, contains, "msgnum:" /dev/null +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log diff --git a/tests/testsuites/failover-no-rptd.conf b/tests/testsuites/failover-no-rptd.conf new file mode 100644 index 00000000..a46ce116 --- /dev/null +++ b/tests/testsuites/failover-no-rptd.conf @@ -0,0 +1,9 @@ +# see the equally-named .sh file for details +$IncludeConfig diag-common.conf + +$RepeatedMsgReduction on + +# second action should never execute +:msg, contains, "msgnum:" /dev/null +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log diff --git a/tests/testsuites/failover-rptd.conf b/tests/testsuites/failover-rptd.conf new file mode 100644 index 00000000..d3553dbb --- /dev/null +++ b/tests/testsuites/failover-rptd.conf @@ -0,0 +1,10 @@ +# see the equally-named .sh file for details +$IncludeConfig diag-common.conf + +$RepeatedMsgReduction on + +$template outfmt,"%msg:F,58:2%\n" +# note: the target server shall not be available! +:msg, contains, "msgnum:" @@127.0.0.1:13514 +$ActionExecOnlyWhenPreviousIsSuspended on +& ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/rcvr_fail_restore_rcvr.conf b/tests/testsuites/rcvr_fail_restore_rcvr.conf new file mode 100644 index 00000000..38ebad29 --- /dev/null +++ b/tests/testsuites/rcvr_fail_restore_rcvr.conf @@ -0,0 +1,8 @@ +$IncludeConfig diag-common2.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# then SENDER sends to this port (not tcpflood!) +$InputTCPServerRun 13515 + +$template outfmt,"%msg:F,58:2%\n" +:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt diff --git a/tests/testsuites/rcvr_fail_restore_sender.conf b/tests/testsuites/rcvr_fail_restore_sender.conf new file mode 100644 index 00000000..6b11aa4a --- /dev/null +++ b/tests/testsuites/rcvr_fail_restore_sender.conf @@ -0,0 +1,15 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +# this listener is for message generation by the test framework! +$InputTCPServerRun 13514 + +$WorkDirectory test-spool +$MainMsgQueueMaxFileSize 1g +$MainMsgQueueFileName mainq + +# we use the shortest resume interval a) to let the test not run too long +# and b) make sure some retries happen before the reconnect +$ActionResumeInterval 1 +$ActionResumeRetryCount -1 +*.* @@127.0.0.1:13515 diff --git a/tests/testsuites/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1 index 23836c9f..8a66d7ba 100644 --- a/tests/testsuites/rfc5424-1.parse1 +++ b/tests/testsuites/rfc5424-1.parse1 @@ -1,3 +1,3 @@ #Example from RFC5424, section 6.5 / sample 1 <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 -34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,BOM'su root' failed for lonvick on /dev/pts/8 diff --git a/tests/testsuites/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1 index a86fbc35..f5449e68 100644 --- a/tests/testsuites/rfc5424-2.parse1 +++ b/tests/testsuites/rfc5424-2.parse1 @@ -1,4 +1,4 @@ <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. -165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. +165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],%% It's time to make the do-nuts. #Example from RFC5424, section 6.5 / sample 2 #Only the first two lines are important, you may place anything behind them! diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c index 2bd18042..b06f1347 100644 --- a/tools/pmrfc5424.c +++ b/tools/pmrfc5424.c @@ -142,7 +142,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr * structured data. There may also be \] inside the structured data, which * do NOT terminate an element. */ - if(lenStr == 0 || *p2parse != '[') + if(lenStr == 0 || (*p2parse != '[' && *p2parse != '-')) return 1; /* this is NOT structured data! */ if(*p2parse == '-') { /* empty structured data? */ @@ -211,7 +211,6 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr * * rger, 2005-11-24 */ -//static int parseRFCSyslogMsg(msg_t *pMsg, int flags) BEGINparse uchar *p2parse; uchar *pBuf = NULL; diff --git a/tools/syslogd.c b/tools/syslogd.c index fb2f1f21..44c88624 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -796,7 +796,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) DBGPRINTF("flush %s: repeated %d times, %d sec.\n", module.GetStateName(pAction->pMod), pAction->f_prevcount, repeatinterval[pAction->f_repeatcount]); - actionWriteToAction(pAction); + actionWriteToAction(pAction, NULL, 0); BACKOFF(pAction); } UnlockObj(pAction); |