From ff6232d2bee06fbbc25ad83a1cf3bb798884679a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Jun 2009 08:12:14 +0200 Subject: bugfix: error messages were not emitted to stderr in forked mode stderr and stdo are now kept open across fork() --- ChangeLog | 2 ++ tools/syslogd.c | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index e9075ab0..e88cfbff 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 3.22.1 [v3-stable] (rgerhards), 2009-04-?? +- bugfix: error messages were not emitted to stderr in forked mode + (stderr and stdo are now kept open across forks) - bugfix: internal messages were emitted to whatever file had fd2 when rsyslogd ran in forked mode (as usual!) Thanks to varmojfekoj for the patch diff --git a/tools/syslogd.c b/tools/syslogd.c index 357f6699..0f63f1bc 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -925,7 +925,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) * permits us to process unmodified config files which otherwise contain a * supressor statement. */ - if(((Debug || NoFork) && bErrMsgToStderr) || iConfigVerify) { + if(bErrMsgToStderr || iConfigVerify) { fprintf(stderr, "rsyslogd: %s\n", msg); } @@ -3096,7 +3096,9 @@ doGlblProcessInit(void) exit(1); /* "good" exit - after forking, not diasabling anything */ } num_fds = getdtablesize(); - for (i= 0; i < num_fds; i++) + close(0); + /* we keep stdout and stderr open in case we have to emit something */ + for (i = 3; i < num_fds; i++) (void) close(i); untty(); } @@ -3532,6 +3534,7 @@ finalize_it: */ int main(int argc, char **argv) { + fprintf(stderr, "rsyslogd startup\n"); dbgClassInit(); return realMain(argc, argv); } -- cgit From 7f78b2ef65634d4320325cca4793caa3d50ef777 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Jun 2009 08:13:41 +0200 Subject: removed forgotten debug message --- tools/syslogd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index 0f63f1bc..1bbf1904 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3534,7 +3534,6 @@ finalize_it: */ int main(int argc, char **argv) { - fprintf(stderr, "rsyslogd startup\n"); dbgClassInit(); return realMain(argc, argv); } -- cgit From 6f0db63e9b962edf6305860b608500e8c650b71b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 15:13:47 +0200 Subject: milestone: input-side multiSubmit capability ... commit before I try to touch the queue side ;) --- runtime/rsyslog.h | 13 +++++++++++++ runtime/stream.c | 2 +- tcps_sess.c | 34 ++++++++++++++++++++++++++-------- tools/syslogd.c | 26 ++++++++++++++++++++++++-- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index e81c9eef..ea8a5222 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -127,6 +127,19 @@ typedef enum { } fiop_t; +/* multi-submit support. + * This is done via a simple data structure, which holds the number of elements + * as well as an array of to-be-submitted messages. + * rgerhards, 2009-06-16 + */ +typedef struct multi_submit_s multi_submit_t; +struct multi_submit_s { + short maxElem; /* maximum number of Elements */ + short nElem; /* current number of Elements, points to the next one FREE */ + msg_t **ppMsgs; +}; + + #ifndef _PATH_CONSOLE #define _PATH_CONSOLE "/dev/console" #endif diff --git a/runtime/stream.c b/runtime/stream.c index f13258b5..8cbe0297 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -188,7 +188,7 @@ finalize_it: static rsRetVal doPhysOpen(strm_t *pThis) { - int iFlags; + int iFlags = 0; DEFiRet; ISOBJ_TYPE_assert(pThis, strm); diff --git a/tcps_sess.c b/tcps_sess.c index f887e702..379faeab 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -220,7 +220,7 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime) +defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) { msg_t *pMsg; DEFiRet; @@ -245,7 +245,15 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG MsgSetRcvFrom(pMsg, pThis->fromHost); MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); - CHKiRet(submitMsg(pMsg)); + + if(pMultiSub == NULL) { + CHKiRet(submitMsg(pMsg)); + } else { + pMultiSub->ppMsgs[pMultiSub->nElem++] = pMsg; + if(pMultiSub->nElem == pMultiSub->maxElem) + CHKiRet(multiSubmitMsg(pMultiSub)); + } + finalize_it: /* reset status variables */ @@ -299,7 +307,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); datetime.getCurrTime(&stTime, &ttGenTime); - defaultDoSubmitMessage(pThis, &stTime, ttGenTime); + defaultDoSubmitMessage(pThis, &stTime, ttGenTime, NULL); } finalize_it: @@ -334,7 +342,7 @@ Close(tcps_sess_t *pThis) * rgerhards, 2008-03-14 */ static rsRetVal -processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime) +processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -380,7 +388,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - defaultDoSubmitMessage(pThis, stTime, ttGenTime); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -391,7 +399,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - defaultDoSubmitMessage(pThis, stTime, ttGenTime); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -408,7 +416,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - defaultDoSubmitMessage(pThis, stTime, ttGenTime); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); pThis->inputState = eAtStrtFram; } } @@ -433,9 +441,12 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt * this *is* the *correct* reception step for all the data we received, because * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ +#define NUM_MULTISUB 128 static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { + multi_submit_t multiSub; + msg_t *pMsgs[NUM_MULTISUB]; struct syslogTime stTime; time_t ttGenTime; char *pEnd; @@ -446,18 +457,25 @@ DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) assert(iLen > 0); datetime.getCurrTime(&stTime, &ttGenTime); + multiSub.ppMsgs = pMsgs; + multiSub.maxElem = NUM_MULTISUB; + multiSub.nElem = 0; /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ iNbrTimeUsed = 0; /* full time query */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime)); + CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime, &multiSub)); } + /* submit anything that was not yet submitted */ + CHKiRet(multiSubmitMsg(&multiSub)); + finalize_it: RETiRet; } +#undef NUM_MULTISUB /* queryInterface function diff --git a/tools/syslogd.c b/tools/syslogd.c index ea8eff7a..4d2839a1 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1345,8 +1345,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } -/* submit a fully created message to the main message queue. The message is - * fully processed and parsed, so no parsing at all happens. This is primarily +/* submit a message to the main message queue. This is primarily * a hook to prevent the need for callers to know about the main message queue * (which may change in the future as we will probably have multiple rule * sets and thus queues...). @@ -1366,6 +1365,29 @@ submitMsg(msg_t *pMsg) } +/* submit multiple messages at once, very similar to submitMsg, just + * for multi_submit_t. + * rgerhards, 2009-06-16 + */ +rsRetVal +multiSubmitMsg(multi_submit_t *pMultiSub) +{ + int i; + DEFiRet; + assert(pMultiSub != NULL); + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { +dbgprintf("multiSubmitMsg, index %d\n", i); + MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); + qqueueEnqObj(pMsgQueue, pMultiSub->ppMsgs[i]->flowCtlType, (void*) pMultiSub->ppMsgs[i]); + } + + pMultiSub->nElem = 0; + + RETiRet; +} + + /* Log a message to the appropriate log files, users, etc. based on * the priority. * rgerhards 2004-11-08: actually, this also decodes all but the PRI part. -- cgit From c4e18c5bab66537ee9c8fb482edce62c7c39c247 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 15:43:22 +0200 Subject: implemented first version of multi-enqueue support, queue side --- runtime/queue.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tcps_sess.c | 2 +- tools/syslogd.c | 2 +- 3 files changed, 128 insertions(+), 2 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index aa8e6c21..dbf59210 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -49,6 +49,7 @@ #include "obj.h" #include "wtp.h" #include "wti.h" +#include "msg.h" #include "atomic.h" #ifdef OS_SOLARIS @@ -2230,6 +2231,131 @@ finalize_it: } +/* enqueue a single data object. This currently is a helper to qqueueMultiEnqObj. + * Note that the queue mutex MUST already be locked when this function is called. + * rgerhards, 2009-06-16 + */ +static inline rsRetVal +doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) +{ + DEFiRet; + struct timespec t; + + /* first check if we need to discard this message (which will cause CHKiRet() to exit) + */ + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + + /* then check if we need to add an assistance disk queue */ + if(pThis->bIsDA) + CHKiRet(qqueueChkStrtDA(pThis)); + + /* handle flow control + * There are two different flow control mechanisms: basic and advanced flow control. + * Basic flow control has always been implemented and protects the queue structures + * in that it makes sure no more data is enqueued than the queue is configured to + * support. Enhanced flow control is being added today. There are some sources which + * can easily be stopped, e.g. a file reader. This is the case because it is unlikely + * that blocking those sources will have negative effects (after all, the file is + * continued to be written). Other sources can somewhat be blocked (e.g. the kernel + * log reader or the local log stream reader): in general, nothing is lost if messages + * from these sources are not picked up immediately. HOWEVER, they can not block for + * an extended period of time, as this either causes message loss or - even worse - some + * other bad effects (e.g. unresponsive system in respect to the main system log socket). + * Finally, there are some (few) sources which can not be blocked at all. UDP syslog is + * a prime example. If a UDP message is not received, it is simply lost. So we can't + * do anything against UDP sockets that come in too fast. The core idea of advanced + * flow control is that we take into account the different natures of the sources and + * select flow control mechanisms that fit these needs. This also means, in the end + * result, that non-blockable sources like UDP syslog receive priority in the system. + * It's a side effect, but a good one ;) -- rgerhards, 2008-03-14 + */ + if(flowCtlType == eFLOWCTL_FULL_DELAY) { + while(pThis->iQueueSize >= pThis->iFullDlyMrk) { + dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); + pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */ + } + } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) { + if(pThis->iQueueSize >= pThis->iLightDlyMrk) { + dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); + timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ + pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */ + } + } + + /* from our regular flow control settings, we are now ready to enqueue the object. + * However, we now need to do a check if the queue permits to add more data. If that + * is not the case, basic flow control enters the field, which means we wait for + * the queue to become ready or drop the new message. -- rgerhards, 2008-03-14 + */ + while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize) + || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0 + && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { + dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); + timeoutComp(&t, pThis->toEnq); + if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { + dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); + objDestruct(pUsr); + ABORT_FINALIZE(RS_RET_QUEUE_FULL); + } + } + + /* and finally enqueue the message */ + CHKiRet(qqueueAdd(pThis, pUsr)); + qqueueChkPersist(pThis); // TODO: optimize, do in outer function! + +finalize_it: + RETiRet; +} + +/* enqueue multiple user data elements at once. The aim is to provide a faster interface + * for object submission. Uses the multi_submit_t helper object. + * Please note that this function is not cancel-safe and consequently + * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE + * during its execution. If that is not done, race conditions occur if the + * thread is canceled (most important use case is input module termination). + * rgerhards, 2009-06-16 + */ +rsRetVal +qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub) +{ + int iCancelStateSave; + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pMultiSub != NULL); + + if(pThis->qType != QUEUETYPE_DIRECT) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); + } + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { +dbgprintf("queueMultiEnq: %d\n", i); + CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i])); + } + +finalize_it: + if(pThis->qType != QUEUETYPE_DIRECT) { + /* make sure at least one worker is running. */ + qqueueAdviseMaxWorkers(pThis); + /* and release the mutex */ + d_pthread_mutex_unlock(pThis->mut); + pthread_setcancelstate(iCancelStateSave, NULL); + dbgoprint((obj_t*) pThis, "MultiEnqObj advised worker start\n"); + /* the following pthread_yield is experimental, but brought us performance + * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 + * rgerhards, 2008-10-09 + * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 + */ + if(pThis->bOptimizeUniProc) + pthread_yield(); + } + + RETiRet; +} + + /* set queue mode to enqueue only or not * There is one subtle issue: this method may be called during queue * construction or while it is running. In the former case, the queue diff --git a/tcps_sess.c b/tcps_sess.c index 379faeab..9353571c 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -441,7 +441,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt * this *is* the *correct* reception step for all the data we received, because * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ -#define NUM_MULTISUB 128 +#define NUM_MULTISUB 1024 static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { diff --git a/tools/syslogd.c b/tools/syslogd.c index 4d2839a1..e3cd54f7 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1379,9 +1379,9 @@ multiSubmitMsg(multi_submit_t *pMultiSub) for(i = 0 ; i < pMultiSub->nElem ; ++i) { dbgprintf("multiSubmitMsg, index %d\n", i); MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); - qqueueEnqObj(pMsgQueue, pMultiSub->ppMsgs[i]->flowCtlType, (void*) pMultiSub->ppMsgs[i]); } + iRet = qqueueMultiEnqObj(pMsgQueue, pMultiSub); pMultiSub->nElem = 0; RETiRet; -- cgit From 8a37736287fd4f7e4fd0c0190aabac11dff19af2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 19:18:41 +0200 Subject: some cleanup --- runtime/queue.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index dbf59210..46a3a971 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -63,7 +63,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(strm) /* forward-definitions */ -rsRetVal qqueueChkPersist(qqueue_t *pThis); +static rsRetVal qqueueChkPersist(qqueue_t *pThis); static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); static rsRetVal qqueueRateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); @@ -1984,10 +1984,8 @@ finalize_it: * abide to our regular call interface)... * rgerhards, 2008-01-13 */ -rsRetVal qqueueChkPersist(qqueue_t *pThis) +static rsRetVal qqueueChkPersist(qqueue_t *pThis) { - DEFiRet; - ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { @@ -1995,7 +1993,7 @@ rsRetVal qqueueChkPersist(qqueue_t *pThis) pThis->iUpdsSincePersist = 0; } - RETiRet; + return RS_RET_OK; } @@ -2301,7 +2299,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis); // TODO: optimize, do in outer function! + qqueueChkPersist(pThis); // TODO: optimize, do in outer function! (but we need parts from v5?) finalize_it: RETiRet; -- cgit From 1e30f67584ca0770e1e5b88ea75da7f9bc1022df Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 19:32:53 +0200 Subject: added basic plumbing to support message properties separate from message will fill this with live somewhat later, noticed I need to do some stage work first (at least this is useful). --- runtime/Makefile.am | 2 + runtime/prop.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/prop.h | 46 +++++++++++++++++++ runtime/rsyslog.h | 1 + 4 files changed, 175 insertions(+) create mode 100644 runtime/prop.c create mode 100644 runtime/prop.h diff --git a/runtime/Makefile.am b/runtime/Makefile.am index c2ef7cfa..14abe722 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -73,6 +73,8 @@ librsyslog_la_SOURCES = \ ruleset.h \ rule.c \ rule.h \ + prop.c \ + prop.h \ cfsysline.c \ cfsysline.h \ \ diff --git a/runtime/prop.c b/runtime/prop.c new file mode 100644 index 00000000..4130d4b7 --- /dev/null +++ b/runtime/prop.c @@ -0,0 +1,126 @@ +/* prop.c - rsyslog's prop object + * + * This object is meant to support message properties that are stored + * seperately from the message. The main intent is to support properties + * that are "constant" during a period of time, so that many messages may + * contain a reference to the same property. It is important, though, that + * properties are destroyed when they are no longer needed. + * + * Please note that this is a performance-critical part of the software and + * as such we may use some methods in here which do not look elegant, but + * which are fast... + * + * Module begun 2009-06-17 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "prop.h" + +/* static data */ +DEFobjStaticHelpers + + +/* Standard-Constructor + */ +BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(prop) + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +propConstructFinalize(prop_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + +finalize_it: + RETiRet; +} + + +/* destructor for the prop object */ +BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(prop) +ENDobjDestruct(prop) + + +/* debugprint for the prop object */ +BEGINobjDebugPrint(prop) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(prop) + dbgprintf("prop object %p - no further debug info implemented\n", pThis); +ENDobjDebugPrint(prop) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(prop) +CODESTARTobjQueryInterface(prop) + if(pIf->ifVersion != propCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = propConstruct; + pIf->ConstructFinalize = propConstructFinalize; + pIf->Destruct = propDestruct; + pIf->DebugPrint = propDebugPrint; + +finalize_it: +ENDobjQueryInterface(prop) + + +/* Exit the prop class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(prop, OBJ_IS_CORE_MODULE) /* class, version */ +// objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(prop) + + +/* Initialize the prop class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(prop, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ +// CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, propDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, propConstructFinalize); +ENDObjClassInit(prop) + +/* vi:set ai: + */ diff --git a/runtime/prop.h b/runtime/prop.h new file mode 100644 index 00000000..7fc466b5 --- /dev/null +++ b/runtime/prop.h @@ -0,0 +1,46 @@ +/* The prop object. + * + * This implements props within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_PROP_H +#define INCLUDED_PROP_H + +/* the prop object */ +struct prop_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ +}; + +/* interfaces */ +BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(prop); + rsRetVal (*Construct)(prop_t **ppThis); + rsRetVal (*ConstructFinalize)(prop_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(prop_t **ppThis); +ENDinterface(prop) +#define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(prop); + +#endif /* #ifndef INCLUDED_PROP_H */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ea8a5222..2e0a4150 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -79,6 +79,7 @@ typedef struct nsdsel_gtls_s nsdsel_gtls_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; typedef struct msg msg_t; +typedef struct prop_s prop_t; typedef struct interface_s interface_t; typedef struct objInfo_s objInfo_t; typedef enum rsRetVal_ rsRetVal; /**< friendly type for global return value */ -- cgit From 56e462610db0dc71cfc2e4af17d1eb27bd67fae7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 12:56:58 +0200 Subject: further optimized message object pri, facility and severity string generation simplified --- dirty.h | 1 + runtime/msg.c | 345 ++++++++++++++++++++++++++++++++++++------------------- runtime/msg.h | 18 +-- runtime/parser.c | 5 - runtime/queue.h | 1 + tools/syslogd.c | 1 - 6 files changed, 234 insertions(+), 137 deletions(-) diff --git a/dirty.h b/dirty.h index 513886b5..8a0cbc80 100644 --- a/dirty.h +++ b/dirty.h @@ -27,6 +27,7 @@ #ifndef DIRTY_H_INCLUDED #define DIRTY_H_INCLUDED 1 +rsRetVal multiSubmitMsg(multi_submit_t *pMultiSub); rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); diff --git a/runtime/msg.c b/runtime/msg.c index 65041a31..8230a340 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -55,53 +55,212 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) -static syslogCODE rs_prioritynames[] = - { - { "alert", LOG_ALERT }, - { "crit", LOG_CRIT }, - { "debug", LOG_DEBUG }, - { "emerg", LOG_EMERG }, - { "err", LOG_ERR }, - { "error", LOG_ERR }, /* DEPRECATED */ - { "info", LOG_INFO }, - { "none", INTERNAL_NOPRI }, /* INTERNAL */ - { "notice", LOG_NOTICE }, - { "panic", LOG_EMERG }, /* DEPRECATED */ - { "warn", LOG_WARNING }, /* DEPRECATED */ - { "warning", LOG_WARNING }, - { NULL, -1 } - }; - -#ifndef LOG_AUTHPRIV -# define LOG_AUTHPRIV LOG_AUTH -#endif -static syslogCODE rs_facilitynames[] = - { - { "auth", LOG_AUTH }, - { "authpriv", LOG_AUTHPRIV }, - { "cron", LOG_CRON }, - { "daemon", LOG_DAEMON }, -#if defined(LOG_FTP) - {"ftp", LOG_FTP}, -#endif - { "kern", LOG_KERN }, - { "lpr", LOG_LPR }, - { "mail", LOG_MAIL }, - { "news", LOG_NEWS }, - { "security", LOG_AUTH }, /* DEPRECATED */ - { "syslog", LOG_SYSLOG }, - { "user", LOG_USER }, - { "uucp", LOG_UUCP }, - { "local0", LOG_LOCAL0 }, - { "local1", LOG_LOCAL1 }, - { "local2", LOG_LOCAL2 }, - { "local3", LOG_LOCAL3 }, - { "local4", LOG_LOCAL4 }, - { "local5", LOG_LOCAL5 }, - { "local6", LOG_LOCAL6 }, - { "local7", LOG_LOCAL7 }, - { NULL, -1 } - }; +static struct { + uchar *pszName; + short lenName; +} syslog_pri_names[192] = { + { UCHAR_CONSTANT("0"), 3}, + { UCHAR_CONSTANT("1"), 3}, + { UCHAR_CONSTANT("2"), 3}, + { UCHAR_CONSTANT("3"), 3}, + { UCHAR_CONSTANT("4"), 3}, + { UCHAR_CONSTANT("5"), 3}, + { UCHAR_CONSTANT("6"), 3}, + { UCHAR_CONSTANT("7"), 3}, + { UCHAR_CONSTANT("8"), 3}, + { UCHAR_CONSTANT("9"), 3}, + { UCHAR_CONSTANT("10"), 4}, + { UCHAR_CONSTANT("11"), 4}, + { UCHAR_CONSTANT("12"), 4}, + { UCHAR_CONSTANT("13"), 4}, + { UCHAR_CONSTANT("14"), 4}, + { UCHAR_CONSTANT("15"), 4}, + { UCHAR_CONSTANT("16"), 4}, + { UCHAR_CONSTANT("17"), 4}, + { UCHAR_CONSTANT("18"), 4}, + { UCHAR_CONSTANT("19"), 4}, + { UCHAR_CONSTANT("20"), 4}, + { UCHAR_CONSTANT("21"), 4}, + { UCHAR_CONSTANT("22"), 4}, + { UCHAR_CONSTANT("23"), 4}, + { UCHAR_CONSTANT("24"), 4}, + { UCHAR_CONSTANT("25"), 4}, + { UCHAR_CONSTANT("26"), 4}, + { UCHAR_CONSTANT("27"), 4}, + { UCHAR_CONSTANT("28"), 4}, + { UCHAR_CONSTANT("29"), 4}, + { UCHAR_CONSTANT("30"), 4}, + { UCHAR_CONSTANT("31"), 4}, + { UCHAR_CONSTANT("32"), 4}, + { UCHAR_CONSTANT("33"), 4}, + { UCHAR_CONSTANT("34"), 4}, + { UCHAR_CONSTANT("35"), 4}, + { UCHAR_CONSTANT("36"), 4}, + { UCHAR_CONSTANT("37"), 4}, + { UCHAR_CONSTANT("38"), 4}, + { UCHAR_CONSTANT("39"), 4}, + { UCHAR_CONSTANT("40"), 4}, + { UCHAR_CONSTANT("41"), 4}, + { UCHAR_CONSTANT("42"), 4}, + { UCHAR_CONSTANT("43"), 4}, + { UCHAR_CONSTANT("44"), 4}, + { UCHAR_CONSTANT("45"), 4}, + { UCHAR_CONSTANT("46"), 4}, + { UCHAR_CONSTANT("47"), 4}, + { UCHAR_CONSTANT("48"), 4}, + { UCHAR_CONSTANT("49"), 4}, + { UCHAR_CONSTANT("50"), 4}, + { UCHAR_CONSTANT("51"), 4}, + { UCHAR_CONSTANT("52"), 4}, + { UCHAR_CONSTANT("53"), 4}, + { UCHAR_CONSTANT("54"), 4}, + { UCHAR_CONSTANT("55"), 4}, + { UCHAR_CONSTANT("56"), 4}, + { UCHAR_CONSTANT("57"), 4}, + { UCHAR_CONSTANT("58"), 4}, + { UCHAR_CONSTANT("59"), 4}, + { UCHAR_CONSTANT("60"), 4}, + { UCHAR_CONSTANT("61"), 4}, + { UCHAR_CONSTANT("62"), 4}, + { UCHAR_CONSTANT("63"), 4}, + { UCHAR_CONSTANT("64"), 4}, + { UCHAR_CONSTANT("65"), 4}, + { UCHAR_CONSTANT("66"), 4}, + { UCHAR_CONSTANT("67"), 4}, + { UCHAR_CONSTANT("68"), 4}, + { UCHAR_CONSTANT("69"), 4}, + { UCHAR_CONSTANT("70"), 4}, + { UCHAR_CONSTANT("71"), 4}, + { UCHAR_CONSTANT("72"), 4}, + { UCHAR_CONSTANT("73"), 4}, + { UCHAR_CONSTANT("74"), 4}, + { UCHAR_CONSTANT("75"), 4}, + { UCHAR_CONSTANT("76"), 4}, + { UCHAR_CONSTANT("77"), 4}, + { UCHAR_CONSTANT("78"), 4}, + { UCHAR_CONSTANT("79"), 4}, + { UCHAR_CONSTANT("80"), 4}, + { UCHAR_CONSTANT("81"), 4}, + { UCHAR_CONSTANT("82"), 4}, + { UCHAR_CONSTANT("83"), 4}, + { UCHAR_CONSTANT("84"), 4}, + { UCHAR_CONSTANT("85"), 4}, + { UCHAR_CONSTANT("86"), 4}, + { UCHAR_CONSTANT("87"), 4}, + { UCHAR_CONSTANT("88"), 4}, + { UCHAR_CONSTANT("89"), 4}, + { UCHAR_CONSTANT("90"), 4}, + { UCHAR_CONSTANT("91"), 4}, + { UCHAR_CONSTANT("92"), 4}, + { UCHAR_CONSTANT("93"), 4}, + { UCHAR_CONSTANT("94"), 4}, + { UCHAR_CONSTANT("95"), 4}, + { UCHAR_CONSTANT("96"), 4}, + { UCHAR_CONSTANT("97"), 4}, + { UCHAR_CONSTANT("98"), 4}, + { UCHAR_CONSTANT("99"), 4}, + { UCHAR_CONSTANT("100"), 5}, + { UCHAR_CONSTANT("101"), 5}, + { UCHAR_CONSTANT("102"), 5}, + { UCHAR_CONSTANT("103"), 5}, + { UCHAR_CONSTANT("104"), 5}, + { UCHAR_CONSTANT("105"), 5}, + { UCHAR_CONSTANT("106"), 5}, + { UCHAR_CONSTANT("107"), 5}, + { UCHAR_CONSTANT("108"), 5}, + { UCHAR_CONSTANT("109"), 5}, + { UCHAR_CONSTANT("110"), 5}, + { UCHAR_CONSTANT("111"), 5}, + { UCHAR_CONSTANT("112"), 5}, + { UCHAR_CONSTANT("113"), 5}, + { UCHAR_CONSTANT("114"), 5}, + { UCHAR_CONSTANT("115"), 5}, + { UCHAR_CONSTANT("116"), 5}, + { UCHAR_CONSTANT("117"), 5}, + { UCHAR_CONSTANT("118"), 5}, + { UCHAR_CONSTANT("119"), 5}, + { UCHAR_CONSTANT("120"), 5}, + { UCHAR_CONSTANT("121"), 5}, + { UCHAR_CONSTANT("122"), 5}, + { UCHAR_CONSTANT("123"), 5}, + { UCHAR_CONSTANT("124"), 5}, + { UCHAR_CONSTANT("125"), 5}, + { UCHAR_CONSTANT("126"), 5}, + { UCHAR_CONSTANT("127"), 5}, + { UCHAR_CONSTANT("128"), 5}, + { UCHAR_CONSTANT("129"), 5}, + { UCHAR_CONSTANT("130"), 5}, + { UCHAR_CONSTANT("131"), 5}, + { UCHAR_CONSTANT("132"), 5}, + { UCHAR_CONSTANT("133"), 5}, + { UCHAR_CONSTANT("134"), 5}, + { UCHAR_CONSTANT("135"), 5}, + { UCHAR_CONSTANT("136"), 5}, + { UCHAR_CONSTANT("137"), 5}, + { UCHAR_CONSTANT("138"), 5}, + { UCHAR_CONSTANT("139"), 5}, + { UCHAR_CONSTANT("140"), 5}, + { UCHAR_CONSTANT("141"), 5}, + { UCHAR_CONSTANT("142"), 5}, + { UCHAR_CONSTANT("143"), 5}, + { UCHAR_CONSTANT("144"), 5}, + { UCHAR_CONSTANT("145"), 5}, + { UCHAR_CONSTANT("146"), 5}, + { UCHAR_CONSTANT("147"), 5}, + { UCHAR_CONSTANT("148"), 5}, + { UCHAR_CONSTANT("149"), 5}, + { UCHAR_CONSTANT("150"), 5}, + { UCHAR_CONSTANT("151"), 5}, + { UCHAR_CONSTANT("152"), 5}, + { UCHAR_CONSTANT("153"), 5}, + { UCHAR_CONSTANT("154"), 5}, + { UCHAR_CONSTANT("155"), 5}, + { UCHAR_CONSTANT("156"), 5}, + { UCHAR_CONSTANT("157"), 5}, + { UCHAR_CONSTANT("158"), 5}, + { UCHAR_CONSTANT("159"), 5}, + { UCHAR_CONSTANT("160"), 5}, + { UCHAR_CONSTANT("161"), 5}, + { UCHAR_CONSTANT("162"), 5}, + { UCHAR_CONSTANT("163"), 5}, + { UCHAR_CONSTANT("164"), 5}, + { UCHAR_CONSTANT("165"), 5}, + { UCHAR_CONSTANT("166"), 5}, + { UCHAR_CONSTANT("167"), 5}, + { UCHAR_CONSTANT("168"), 5}, + { UCHAR_CONSTANT("169"), 5}, + { UCHAR_CONSTANT("170"), 5}, + { UCHAR_CONSTANT("171"), 5}, + { UCHAR_CONSTANT("172"), 5}, + { UCHAR_CONSTANT("173"), 5}, + { UCHAR_CONSTANT("174"), 5}, + { UCHAR_CONSTANT("175"), 5}, + { UCHAR_CONSTANT("176"), 5}, + { UCHAR_CONSTANT("177"), 5}, + { UCHAR_CONSTANT("178"), 5}, + { UCHAR_CONSTANT("179"), 5}, + { UCHAR_CONSTANT("180"), 5}, + { UCHAR_CONSTANT("181"), 5}, + { UCHAR_CONSTANT("182"), 5}, + { UCHAR_CONSTANT("183"), 5}, + { UCHAR_CONSTANT("184"), 5}, + { UCHAR_CONSTANT("185"), 5}, + { UCHAR_CONSTANT("186"), 5}, + { UCHAR_CONSTANT("187"), 5}, + { UCHAR_CONSTANT("188"), 5}, + { UCHAR_CONSTANT("189"), 5}, + { UCHAR_CONSTANT("190"), 5}, + { UCHAR_CONSTANT("191"), 5} + }; + +/*syslog facility names (as of RFC5424) */ +static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", "syslog", "lpr", + "news", "uucp", "cron", "authpriv", "ftp", "ntp", "audit", + "alert", "clock", "local0", "local1", "local2", "local3", + "local4", "local5", "local6", "local7" }; + +/* table of severity names (in numerical order)*/ +static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" }; /* some forward declarations */ static int getAPPNAMELen(msg_t *pM); @@ -352,9 +511,7 @@ CODESTARTobjDestruct(msg) free(pThis->pszRcvFromIP); free(pThis->pszMSG); free(pThis->pszFacility); - free(pThis->pszFacilityStr); free(pThis->pszSeverity); - free(pThis->pszSeverityStr); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_SecFrac); @@ -441,12 +598,8 @@ msg_t* MsgDup(msg_t* pOld) * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; */ - memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); - pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); - tmpCOPYSZ(SeverityStr); tmpCOPYSZ(Facility); - tmpCOPYSZ(FacilityStr); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); tmpCOPYSZ(TAG); @@ -734,27 +887,24 @@ char *getMSG(msg_t *pM) /* Get PRI value as integer */ static int getPRIi(msg_t *pM) { - assert(pM != NULL); return (pM->iFacility << 3) + (pM->iSeverity); } -/* Get PRI value in text form */ +/* Get PRI value in text form + */ static inline char *getPRI(msg_t *pM) { + /* PRI is a number in the range 0..191. Thus, we use a simple lookup table to obtain the + * string value. It looks a bit clumpsy here in code ;) + */ + int iPRI; + if(pM == NULL) return ""; - /* there are some cases where bufPRI may not contain a valid string, - * and then we need to build it. - */ - MsgLock(pM); - if(pM->bufPRI[0] == '\0') { - snprintf((char*)pM->bufPRI, sizeof(pM->bufPRI), "<%d>", getPRIi(pM)); - } - MsgUnlock(pM); - - return (char*)pM->bufPRI; + iPRI = getPRIi(pM); + return (iPRI > 191) ? "invld" : (char*)syslog_pri_names[iPRI].pszName; } @@ -934,32 +1084,18 @@ static inline char *getSeverity(msg_t *pM) static inline char *getSeverityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverityStr == NULL) { - for(c = rs_prioritynames, val = pM->iSeverity; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverityStr = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = - snprintf((char*)pM->pszSeverityStr, 2, "%d", pM->iSeverity); - } else { - if((pM->pszSeverityStr = (uchar*) strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = strlen((char*)name); - } + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_severity_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverityStr); + + return name; } static inline char *getFacility(msg_t *pM) @@ -983,35 +1119,18 @@ static inline char *getFacility(msg_t *pM) static inline char *getFacilityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacilityStr == NULL) { - for(c = rs_facilitynames, val = pM->iFacility << 3; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacilityStr = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = - snprintf((char*)pM->pszFacilityStr, 12, "%d", val >> 3); - } else { - if((pM->pszFacilityStr = (uchar*)strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = strlen((char*)name); - } - } - MsgUnlock(pM); - return((char*)pM->pszFacilityStr); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_fac_names[pM->iFacility]; + } + + return name; } @@ -1599,15 +1718,11 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) */ char *textpri(char *pRes, size_t pResLen, int pri) { - syslogCODE *c_pri, *c_fac; - assert(pRes != NULL); assert(pResLen > 0); - for (c_fac = rs_facilitynames; c_fac->c_name && !(c_fac->c_val == LOG_FAC(pri)<<3); c_fac++); - for (c_pri = rs_prioritynames; c_pri->c_name && !(c_pri->c_val == LOG_PRI(pri)); c_pri++); - - snprintf (pRes, pResLen, "%s.%s<%d>", c_fac->c_name, c_pri->c_name, pri); + snprintf(pRes, pResLen, "%s.%s<%d>", syslog_fac_names[LOG_FAC(pri)], + syslog_severity_names[LOG_PRI(pri)], pri); return pRes; } diff --git a/runtime/msg.h b/runtime/msg.h index 74ff9e60..a3058349 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,12 +51,12 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; - bool bDoLock; /* use the mutex? */ + bool bDoLock; /* use the mutex? */ + bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ - short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, * it is available when we have a forwarder (e.g. rfc3195d) using local @@ -66,15 +66,9 @@ struct msg { short iSeverity; /* the severity 0..7 */ uchar *pszSeverity; /* severity as string... */ int iLenSeverity; /* ... and its length. */ - uchar *pszSeverityStr; /* severity name... */ - int iLenSeverityStr; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ uchar *pszFacility; /* Facility as string... */ int iLenFacility; /* ... and its length. */ - uchar *pszFacilityStr; /* facility name... */ - int iLenFacilityStr; /* ... and its length. */ - uchar bufPRI[5]; /* PRI as string */ - int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we * need to preserve cryptographic verifiers. @@ -83,8 +77,6 @@ struct msg { int iLenRawMsg; /* length of raw message */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszUxTradMsg; /* the traditional UNIX message */ - int iLenUxTradMsg;/* Length of the traditional UNIX message */ uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ @@ -122,7 +114,6 @@ struct msg { char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ - /* now follow fixed-size buffers to safe some time otherwise used for allocs */ }; @@ -182,11 +173,6 @@ char *getProgramName(msg_t *pM); int getProgramNameLen(msg_t *pM); uchar *getRcvFrom(msg_t *pM); -#if 0 -char *getUxTradMsg(msg_t *pM); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); -#endif - /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains * us quite some performance and is fully abstracted using this header file. diff --git a/runtime/parser.c b/runtime/parser.c index 0b45bfd5..c13edb8f 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -284,7 +284,6 @@ rsRetVal parseMsg(msg_t *pMsg) DEFiRet; uchar *msg; int pri; - int iPriText; CHKiRet(sanitizeMessage(pMsg)); @@ -294,7 +293,6 @@ rsRetVal parseMsg(msg_t *pMsg) /* pull PRI */ pri = DEFUPRI; msg = pMsg->pszRawMsg; - iPriText = 0; if(*msg == '<') { /* while we process the PRI, we also fill the PRI textual representation * inside the msg object. This may not be ideal from an OOP point of view, @@ -302,11 +300,8 @@ rsRetVal parseMsg(msg_t *pMsg) */ pri = 0; while(isdigit((int) *++msg)) { - pMsg->bufPRI[iPriText++ % 4] = *msg; /* mod 4 to guard against malformed messages! */ pri = 10 * pri + (*msg - '0'); } - pMsg->bufPRI[iPriText % 4] = '\0'; - pMsg->iLenPRI = iPriText % 4; if(*msg == '>') ++msg; if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) diff --git a/runtime/queue.h b/runtime/queue.h index 07f134aa..5bc03254 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -179,6 +179,7 @@ typedef struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); +rsRetVal qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub); rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); diff --git a/tools/syslogd.c b/tools/syslogd.c index e3cd54f7..9f70ff26 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1377,7 +1377,6 @@ multiSubmitMsg(multi_submit_t *pMultiSub) assert(pMultiSub != NULL); for(i = 0 ; i < pMultiSub->nElem ; ++i) { -dbgprintf("multiSubmitMsg, index %d\n", i); MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); } -- cgit From 34b60b389e463e260f880ca3efc591b4db2cd18a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 13:33:57 +0200 Subject: some more optimization in msg object string handling --- runtime/msg.c | 70 ++++++++++++++++++++++------------------------------------- runtime/msg.h | 21 ++++++++---------- 2 files changed, 35 insertions(+), 56 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 8230a340..8e9d2b71 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -262,6 +262,13 @@ static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", /* table of severity names (in numerical order)*/ static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" }; +/* numerical values as string - this is the most efficient approach to convert severity + * and facility values to a numerical string... -- rgerhars, 2009-06-17 + */ + +static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", + "15", "16", "17", "18", "19", "20", "21", "22", "23" }; + /* some forward declarations */ static int getAPPNAMELen(msg_t *pM); static int getProtocolVersion(msg_t *pM); @@ -510,15 +517,11 @@ CODESTARTobjDestruct(msg) free(pThis->pszRcvFrom); free(pThis->pszRcvFromIP); free(pThis->pszMSG); - free(pThis->pszFacility); - free(pThis->pszSeverity); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_SecFrac); free(pThis->pszRcvdAt_MySQL); free(pThis->pszRcvdAt_PgSQL); - free(pThis->pszTIMESTAMP3164); - free(pThis->pszTIMESTAMP3339); free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); @@ -598,8 +601,6 @@ msg_t* MsgDup(msg_t* pOld) * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; */ - tmpCOPYSZ(Severity); - tmpCOPYSZ(Facility); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); tmpCOPYSZ(TAG); @@ -916,12 +917,10 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) switch(eFmt) { case tplFmtDefault: + case tplFmtRFC3164Date: MsgLock(pM); if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } + pM->pszTIMESTAMP3164 = pM->pszTimestamp3164; datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); } MsgUnlock(pM); @@ -948,24 +947,10 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) } MsgUnlock(pM); return(pM->pszTIMESTAMP_PgSQL); - case tplFmtRFC3164Date: - MsgLock(pM); - if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } - datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); - } - MsgUnlock(pM); - return(pM->pszTIMESTAMP3164); case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszTIMESTAMP3339 == NULL) { - if((pM->pszTIMESTAMP3339 = malloc(33)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ - } + pM->pszTIMESTAMP3339 = pM->pszTimestamp3339; datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339, 33); } MsgUnlock(pM); @@ -1067,18 +1052,18 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) static inline char *getSeverity(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverity == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverity = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverity = - snprintf((char*)pM->pszSeverity, 2, "%d", pM->iSeverity); + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_number_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverity); + + return name; } @@ -1100,21 +1085,18 @@ static inline char *getSeverityStr(msg_t *pM) static inline char *getFacility(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacility == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacility = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacility = - snprintf((char*)pM->pszFacility, 12, "%d", pM->iFacility); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_number_names[pM->iFacility]; } - MsgUnlock(pM); - return((char*)pM->pszFacility); + + return name; } static inline char *getFacilityStr(msg_t *pM) diff --git a/runtime/msg.h b/runtime/msg.h index a3058349..703fdd9f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -50,12 +50,12 @@ */ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because + once data has entered the queue, this property is no longer needed. */ pthread_mutexattr_t mutAttr; + pthread_mutex_t mut; bool bDoLock; /* use the mutex? */ bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ - pthread_mutex_t mut; - flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because - once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -64,17 +64,11 @@ struct msg { * resolve all these issues... rgerhards, 2005-10-06 */ short iSeverity; /* the severity 0..7 */ - uchar *pszSeverity; /* severity as string... */ - int iLenSeverity; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ - uchar *pszFacility; /* Facility as string... */ - int iLenFacility; /* ... and its length. */ - uchar *pszRawMsg; /* message as it was received on the - * wire. This is important in case we - * need to preserve cryptographic verifiers. - */ - short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ + uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we + * need to preserve cryptographic verifiers. */ int iLenRawMsg; /* length of raw message */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ uchar *pszTAG; /* pointer to tag value */ @@ -114,6 +108,9 @@ struct msg { char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ + /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + char pszTimestamp3164[16]; + char pszTimestamp3339[33]; }; -- cgit From d2d54013aebb756169182ed8716b142d27134a70 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 15:22:13 +0200 Subject: going forward in moving string-handling functions to new interface... --- parse.c | 18 ++++++------- runtime/ctok.c | 17 ++++++------ runtime/msg.c | 20 +++++++-------- runtime/nsd_gtls.c | 34 ++++++++++++------------ runtime/obj.c | 16 ++++++------ runtime/prop.c | 4 +-- runtime/stream.c | 6 ++--- runtime/stringbuf.c | 74 +++++++++++++++++++++++++++++++---------------------- runtime/stringbuf.h | 22 +++++++++------- runtime/vm.c | 12 +++++---- runtime/vmop.c | 2 +- tests/nettester.c | 4 ++- tools/syslogd.c | 2 +- 13 files changed, 126 insertions(+), 105 deletions(-) diff --git a/parse.c b/parse.c index 87c67676..5288c8b4 100644 --- a/parse.c +++ b/parse.c @@ -256,7 +256,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != cDelim) { - CHKiRet(rsCStrAppendChar(pCStr, bConvLower ? tolower(*pC) : *pC)); + CHKiRet(cstrAppendChar(pCStr, bConvLower ? tolower(*pC) : *pC)); ++pThis->iCurrPos; ++pC; } @@ -271,7 +271,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim CHKiRet(cstrFinalize(pCStr)); if(bTrimTrailing) { - CHKiRet(rsCStrTrimTrailingWhiteSpace(pCStr)); + CHKiRet(cstrTrimTrailingWhiteSpace(pCStr)); } /* done! */ @@ -313,23 +313,23 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; /* OK, we most probably can obtain a value... */ - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); - while(pThis->iCurrPos < rsCStrLen(pThis->pCStr)) { + while(pThis->iCurrPos < cstrLen(pThis->pCStr)) { if(*pC == '"') { break; /* we are done! */ } else if(*pC == '\\') { ++pThis->iCurrPos; ++pC; - if(pThis->iCurrPos < rsCStrLen(pThis->pCStr)) { + if(pThis->iCurrPos < cstrLen(pThis->pCStr)) { /* in this case, we copy the escaped character * to the output buffer (but do not rely on this, * we might later introduce other things, like \007! */ - CHKiRet(rsCStrAppendChar(pCStr, *pC)); + CHKiRet(cstrAppendChar(pCStr, *pC)); } } else { /* regular character */ - CHKiRet(rsCStrAppendChar(pCStr, *pC)); + CHKiRet(cstrAppendChar(pCStr, *pC)); } ++pThis->iCurrPos; ++pC; @@ -339,7 +339,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) ++pThis->iCurrPos; /* 'eat' trailing quote */ } else { /* error - improperly quoted string! */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); ABORT_FINALIZE(RS_RET_MISSING_TRAIL_QUOTE); } @@ -352,7 +352,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) finalize_it: if(iRet != RS_RET_OK) { if(pCStr != NULL) - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); } RETiRet; diff --git a/runtime/ctok.c b/runtime/ctok.c index 263e656c..6f5f0273 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -258,13 +258,13 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_MSGVAR; } - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); /* this loop is quite simple, a variable name is terminated when a non-supported * character is detected. Note that we currently permit a numerical digit as the * first char, which is not permitted by ABNF. -- rgerhards, 2009-03-10 */ while(isalpha(c) || isdigit(c) || (c == '_') || (c == '-')) { - CHKiRet(rsCStrAppendChar(pstrVal, tolower(c))); + CHKiRet(cstrAppendChar(pstrVal, tolower(c))); CHKiRet(ctokGetCharFromStream(pThis, &c)); } CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ @@ -277,7 +277,7 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -301,20 +301,20 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_SIMPSTR; - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); CHKiRet(ctokGetCharFromStream(pThis, &c)); /* while we are in escape mode (had a backslash), no sequence * terminates the loop. If outside, it is terminated by a single quote. */ while(bInEsc || c != '\'') { if(bInEsc) { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); bInEsc = 0; } else { if(c == '\\') { bInEsc = 1; } else { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); } } CHKiRet(ctokGetCharFromStream(pThis, &c)); @@ -327,7 +327,7 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -519,8 +519,9 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) CHKiRet(ctokUngetCharFromStream(pThis, c)); pToken->tok = ctok_FUNCTION; /* fill function name */ - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); CHKiRet(rsCStrSetSzStr(pstrVal, szWord)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); } else { /* give up... */ dbgprintf("parser has an invalid word (token) '%s'\n", szWord); diff --git a/runtime/msg.c b/runtime/msg.c index 8e9d2b71..d9ff2e73 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -730,10 +730,10 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) ++i; /* skip '[' */ /* now obtain the PROCID string... */ - CHKiRet(rsCStrConstruct(&pM->pCSPROCID)); + CHKiRet(cstrConstruct(&pM->pCSPROCID)); rsCStrSetAllocIncrement(pM->pCSPROCID, 16); while((i < pM->iLenTAG) && (pM->pszTAG[i] != ']')) { - CHKiRet(rsCStrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); ++i; } @@ -743,7 +743,7 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) * the buffer and simply return. Note that this is NOT an error * case! */ - rsCStrDestruct(&pM->pCSPROCID); + cstrDestruct(&pM->pCSPROCID); FINALIZE; } @@ -781,14 +781,14 @@ static rsRetVal aquireProgramName(msg_t *pM) /* ok, we do not yet have it. So let's parse the TAG * to obtain it. */ - CHKiRet(rsCStrConstruct(&pM->pCSProgName)); + CHKiRet(cstrConstruct(&pM->pCSProgName)); rsCStrSetAllocIncrement(pM->pCSProgName, 33); for( i = 0 ; (i < pM->iLenTAG) && isprint((int) pM->pszTAG[i]) && (pM->pszTAG[i] != '\0') && (pM->pszTAG[i] != ':') && (pM->pszTAG[i] != '[') && (pM->pszTAG[i] != '/') ; ++i) { - CHKiRet(rsCStrAppendChar(pM->pCSProgName, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSProgName, pM->pszTAG[i])); } CHKiRet(cstrFinalize(pM->pCSProgName)); } @@ -811,8 +811,7 @@ finalize_it: void moveHOSTNAMEtoTAG(msg_t *pM) { assert(pM != NULL); - if(pM->pszTAG != NULL) - free(pM->pszTAG); + free(pM->pszTAG); pM->pszTAG = pM->pszHOSTNAME; pM->iLenTAG = pM->iLenHOSTNAME; pM->pszHOSTNAME = NULL; @@ -1188,11 +1187,11 @@ rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID) ISOBJ_TYPE_assert(pMsg, msg); if(pMsg->pCSPROCID == NULL) { /* we need to obtain the object first */ - CHKiRet(rsCStrConstruct(&pMsg->pCSPROCID)); - rsCStrSetAllocIncrement(pMsg->pCSPROCID, 128); + CHKiRet(cstrConstruct(&pMsg->pCSPROCID)); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSPROCID, (uchar*) pszPROCID); + CHKiRet(cstrFinalize(pMsg->pCSPROCID)); finalize_it: RETiRet; @@ -1221,7 +1220,8 @@ char *getPROCID(msg_t *pM) MsgLock(pM); if(pM->pCSPROCID == NULL) aquirePROCIDFromTAG(pM); - pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSPROCID); + pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) cstrGetSzStrNoNULL(pM->pCSPROCID); + //pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSPROCID); MsgUnlock(pM); return pszRet; } diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 19dc8678..79ceffb3 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -710,16 +710,16 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) } /* we found a common name, now extract it */ - CHKiRet(rsCStrConstruct(&pstrCN)); + CHKiRet(cstrConstruct(&pstrCN)); while(szDN[i] != '\0' && szDN[i] != ',') { if(szDN[i] == '\\') { /* hex escapes are not implemented */ ++i; /* escape char processed */ if(szDN[i] == '\0') ABORT_FINALIZE(RS_RET_CERT_INVALID_DN); - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } else { - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } ++i; /* char processed */ } @@ -734,7 +734,7 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) finalize_it: if(iRet != RS_RET_OK) { if(pstrCN != NULL) - rsCStrDestruct(&pstrCN); + cstrDestruct(&pstrCN); } RETiRet; @@ -761,7 +761,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) size = sizeof(fingerprint); CHKgnutls(gnutls_x509_crt_get_fingerprint(*pCert, GNUTLS_DIG_SHA1, fingerprint, &size)); CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint)); - dbgprintf("peer's certificate SHA1 fingerprint: %s\n", rsCStrGetSzStr(pstrFingerprint)); + dbgprintf("peer's certificate SHA1 fingerprint: %s\n", cstrGetSzStr(pstrFingerprint)); /* now search through the permitted peers to see if we can find a permitted one */ bFoundPositiveMatch = 0; @@ -779,7 +779,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) if(pThis->bReportAuthErr == 1) { errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer fingerprint '%s' unknown - we are " - "not permitted to talk to it", rsCStrGetSzStr(pstrFingerprint)); + "not permitted to talk to it", cstrGetSzStr(pstrFingerprint)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -787,7 +787,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) finalize_it: if(pstrFingerprint != NULL) - rsCStrDestruct(&pstrFingerprint); + cstrDestruct(&pstrFingerprint); RETiRet; } @@ -874,10 +874,10 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) /* if we did not succeed so far, we try the CN part of the DN... */ CHKiRet(gtlsGetCN(pThis, pCert, &pstrCN)); if(pstrCN != NULL) { /* NULL if there was no CN present */ - dbgprintf("gtls now checking auth for CN '%s'\n", rsCStrGetSzStr(pstrCN)); - snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", rsCStrGetSzStr(pstrCN)); + dbgprintf("gtls now checking auth for CN '%s'\n", cstrGetSzStr(pstrCN)); + snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", cstrGetSzStr(pstrCN)); CHKiRet(rsCStrAppendStr(pStr, lnBuf)); - CHKiRet(gtlsChkOnePeerName(pThis, rsCStrGetSzStr(pstrCN), &bFoundPositiveMatch)); + CHKiRet(gtlsChkOnePeerName(pThis, cstrGetSzStr(pstrCN), &bFoundPositiveMatch)); } } @@ -888,7 +888,7 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer name not authorized - " "not permitted to talk to it. Names: %s", - rsCStrGetSzStr(pStr)); + cstrGetSzStr(pStr)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -1010,8 +1010,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) errmsg.LogError(0, NO_ERRCODE, "not permitted to talk to peer, certificate invalid: %s", pszErrCause); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_INVALID); } @@ -1032,8 +1032,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert > ttNow) { errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "not permitted to talk to peer: certificate %d not yet active", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_NOT_YET_ACTIVE); } @@ -1043,8 +1043,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert < ttNow) { errmsg.LogError(0, RS_RET_CERT_EXPIRED, "not permitted to talk to peer: certificate %d expired", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_EXPIRED); } gnutls_x509_crt_deinit(cert); diff --git a/runtime/obj.c b/runtime/obj.c index f2cb447e..6ca05cc4 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -440,11 +440,11 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) assert(ppStr != NULL); - CHKiRet(rsCStrConstruct(&pStr)); + CHKiRet(cstrConstruct(&pStr)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pStr, c)); + CHKiRet(cstrAppendChar(pStr, c)); NEXTC; } CHKiRet(cstrFinalize(pStr)); @@ -453,7 +453,7 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) finalize_it: if(iRet != RS_RET_OK && pStr != NULL) - rsCStrDestruct(&pStr); + cstrDestruct(&pStr); RETiRet; } @@ -508,11 +508,11 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) assert(ppCStr != NULL); assert(iLen >= 0); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); NEXTC; for(i = 0 ; i < iLen ; ++i) { - CHKiRet(rsCStrAppendChar(pCStr, c)); + CHKiRet(cstrAppendChar(pCStr, c)); NEXTC; } CHKiRet(cstrFinalize(pCStr)); @@ -524,7 +524,7 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) finalize_it: if(iRet != RS_RET_OK && pCStr != NULL) - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); RETiRet; } @@ -622,11 +622,11 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) } /* get the property name first */ - CHKiRet(rsCStrConstruct(&pProp->pcsName)); + CHKiRet(cstrConstruct(&pProp->pcsName)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pProp->pcsName, c)); + CHKiRet(cstrAppendChar(pProp->pcsName, c)); NEXTC; } CHKiRet(cstrFinalize(pProp->pcsName)); diff --git a/runtime/prop.c b/runtime/prop.c index 4130d4b7..02be315f 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -55,12 +55,10 @@ ENDobjConstruct(prop) * rgerhards, 2008-01-09 */ static rsRetVal -propConstructFinalize(prop_t *pThis) +propConstructFinalize(prop_t __attribute__((unused)) *pThis) { DEFiRet; ISOBJ_TYPE_assert(pThis, prop); - -finalize_it: RETiRet; } diff --git a/runtime/stream.c b/runtime/stream.c index 8cbe0297..f9859617 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -531,19 +531,19 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr) ASSERT(pThis != NULL); ASSERT(ppCStr != NULL); - CHKiRet(rsCStrConstruct(ppCStr)); + CHKiRet(cstrConstruct(ppCStr)); /* now read the line */ CHKiRet(strmReadChar(pThis, &c)); while(c != '\n') { - CHKiRet(rsCStrAppendChar(*ppCStr, c)); + CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } CHKiRet(cstrFinalize(*ppCStr)); finalize_it: if(iRet != RS_RET_OK && *ppCStr != NULL) - rsCStrDestruct(ppCStr); + cstrDestruct(ppCStr); RETiRet; } diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index a2d9c599..e7fa8c41 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -1,3 +1,4 @@ +#include /* This is the byte-counted string class for rsyslog. It is a replacement * for classical \0 terminated string functions. We introduce it in * the hope it will make the program more secure, obtain some performance @@ -157,7 +158,7 @@ static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { uchar *pNewBuf; - size_t iNewSize; + unsigned short iNewSize; DEFiRet; /* first compute the new size needed */ @@ -224,7 +225,7 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz) /* append the contents of one cstr_t object to another * rgerhards, 2008-02-25 */ -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) { return rsCStrAppendStrWithLen(pThis, pstrAppend->pBuf, pstrAppend->iStrLen); } @@ -245,32 +246,7 @@ finalize_it: } -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c) -{ - DEFiRet; - - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(pThis->iStrLen >= pThis->iBufSize) { - CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ - } - - /* ok, when we reach this, we have sufficient memory */ - *(pThis->pBuf + pThis->iStrLen++) = c; - - /* check if we need to invalidate an sz representation! */ - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - pThis->pszBuf = NULL; - } - -finalize_it: - RETiRet; -} - - -/* NEW VARIANT - * Append a character to the current string object. This may only be done until +/* Append a character to the current string object. This may only be done until * cstrFinalize() is called. * rgerhards, 2009-06-16 */ @@ -392,7 +368,24 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) } -/* NEW VERSION for interface without separate psz buffer! */ +/* Converts the CStr object to a classical sz string and returns that. + * Same restrictions as in cstrGetSzStr() applies (see there!). This + * function here guarantees that a valid string is returned, even if + * the CStr object currently holds a NULL pointer string buffer. If so, + * "" is returned. + * rgerhards 2005-10-19 + * WARNING: The returned pointer MUST NOT be freed, as it may be + * obtained from that constant memory pool (in case of NULL!) + */ +uchar* cstrGetSzStrNoNULL(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + if(pThis->pBuf == NULL) + return (uchar*) ""; + else + return cstrGetSzStr(pThis); +} + /* Returns the cstr data as a classical C sz string. We use that the * Finalizer did properly terminate our string (but we may stil be NULL). * So it is vital that the finalizer is called BEFORe this function here! @@ -497,7 +490,7 @@ void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) * This is due to performance reasons. */ #ifndef NDEBUG -int rsCStrLen(cstr_t *pThis) +int cstrLen(cstr_t *pThis) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); return(pThis->iStrLen); @@ -548,6 +541,27 @@ rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis) return RS_RET_OK; } +/* Trim trailing whitespace from a given string + */ +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis) +{ + register int i; + register uchar *pC; + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + i = pThis->iStrLen; + pC = pThis->pBuf + i - 1; + while(i > 0 && isspace((int)*pC)) { + --pC; + --i; + } + /* i now is the new string length! */ + pThis->iStrLen = i; + pThis->pBuf[pThis->iStrLen] = '0'; /* we always have this space */ + + return RS_RET_OK; +} + /* compare two string objects - works like strcmp(), but operates * on CStr objects. Please note that this version here is * faster in the majority of cases, simply because it can diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index d28aee26..b4fc3f3c 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -1,5 +1,5 @@ -/*! \file stringbuf.h - * \brief The counted string object +/* stringbuf.h + * The counted string object * * This is the byte-counted string class for rsyslog. It is a replacement * for classical \0 terminated string functions. We introduce it in @@ -11,8 +11,7 @@ * \date 2005-09-07 * Initial version begun. * - * All functions in this "class" start with rsCStr (rsyslog Counted String). - * Copyright 2005 + * Copyright 2005-2009 * Rainer Gerhards and Adiscon GmbH. All Rights Reserved. * * This file is part of the rsyslog runtime library. @@ -48,7 +47,7 @@ typedef struct cstr_s uchar *pszBuf; /**< pointer to the sz version of the string (after it has been created )*/ size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ - size_t iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ + unsigned short iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ bool bIsFinalized; /**< is this object finished and ready for use? (a debug aid, may be removed later TODO 2009-06-16) */ } cstr_t; @@ -73,7 +72,6 @@ void rsCStrDestruct(cstr_t **ppThis); * * \param c Character to append to string. */ -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); /** @@ -85,6 +83,7 @@ rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); rsRetVal rsCStrTruncate(cstr_t *pThis, size_t nTrunc); rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis); +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis); /** * Append a string to the buffer. For performance reasons, @@ -143,19 +142,24 @@ rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *cache) void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); + +/* in migration */ +#define rsCStrAppendCStr(pThis, pstrAppend) cstrAppendCStr(pThis, pstrAppend) /* new calling interface */ rsRetVal cstrFinalize(cstr_t *pThis); rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); uchar* cstrGetSzStr(cstr_t *pThis); +uchar* cstrGetSzStrNoNULL(cstr_t *pThis); +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); /* now come inline-like functions */ #ifdef NDEBUG -# define rsCStrLen(x) ((int)((x)->iStrLen)) +# define cstrLen(x) ((int)((x)->iStrLen)) #else - int rsCStrLen(cstr_t *pThis); + int cstrLen(cstr_t *pThis); #endif +#define rsCStrLen(s) cstrLen((s)) #define rsCStrGetBufBeg(x) ((x)->pBuf) diff --git a/runtime/vm.c b/runtime/vm.c index 8cbf9e12..d7cd52d5 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -82,6 +82,7 @@ rsfrAddFunction(uchar *szName, prsf_t rsf) /* unique name, so add to head of list */ CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + CHKiRet(cstrFinalize(pEntry->pName)); pEntry->rsf = rsf; pEntry->pNext = funcRegRoot; funcRegRoot = pEntry; @@ -167,7 +168,7 @@ rsfrRemoveAll(void) while(pEntry != NULL) { pEntryDel = pEntry; pEntry = pEntry->pNext; - rsCStrDestruct(&pEntryDel->pName); + cstrDestruct(&pEntryDel->pName); free(pEntryDel); } funcRegRoot = NULL; @@ -405,6 +406,7 @@ CODESTARTop(STRADD) vmstk.PopString(pThis->pStk, &operand1); CHKiRet(rsCStrAppendCStr(operand1->val.pStr, operand2->val.pStr)); + CHKiRet(cstrFinalize(operand1->val.pStr)); /* we have a result, so let's push it */ vmstk.Push(pThis->pStk, operand1); @@ -554,12 +556,12 @@ rsf_tolower(vmstk_t *pStk, int numOperands) ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); /* pop args and do operaton */ - CHKiRet(rsCStrConstruct(&pcstr)); + CHKiRet(cstrConstruct(&pcstr)); vmstk.PopString(pStk, &operand1); - pSrc = rsCStrGetSzStr(operand1->val.pStr); - iStrlen = strlen((char*)pSrc); + pSrc = cstrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); // TODO: use count from string! while(iStrlen--) { - CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + CHKiRet(cstrAppendChar(pcstr, tolower(*pSrc++))); } /* Store result and cleanup */ diff --git a/runtime/vmop.c b/runtime/vmop.c index acacfc9e..ea627220 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -125,7 +125,7 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) if(pThis->operand.pVar != NULL) CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); } - CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); + CHKiRet(cstrAppendChar(pstrPrg, '\n')); finalize_it: RETiRet; diff --git a/tests/nettester.c b/tests/nettester.c index 566f553b..b7b3f892 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -321,7 +321,9 @@ doTests(int fd, char *files) if(ret == 0) { if(verbose) printf("successfully completed\n"); } else { - if(verbose) printf("failed!\n"); + if(!verbose) + printf("test '%s' ", testFile); + printf("failed!\n"); ++iFailed; } } diff --git a/tools/syslogd.c b/tools/syslogd.c index 9f70ff26..cfecfd83 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1051,7 +1051,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult) return 0; } -/* parse a RFC-formatted syslog message. This function returns +/* parse a RFC5424-formatted syslog message. This function returns * 0 if processing of the message shall continue and 1 if something * went wrong and this messe should be ignored. This function has been * implemented in the effort to support syslog-protocol. Please note that -- cgit From bca66bc1399d1b9bcc370c64b5a8befb2244695d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 11:10:28 +0200 Subject: some minor bugfixes - bugfix: invalid error message issued if $inlcudeConfig was on an empty set of files (e.g. *.conf, where none such files existed) thanks to Michael Biebl for reporting this bug - bugfix: when run in foreground (but not in debug mode), a debug message ("DoDie called") was emitted at shutdown. Removed. thanks to Michael Biebl for reporting this bug - bugfix: some garbagge was emitted to stderr on shutdown. This garbage consisted of file names, which were written during startup (key point: not a pointer error) thanks to Michael Biebl for reporting this bug - bugfix: startup and shutdown message were emitted to stdout thanks to Michael Biebl for reporting this bug --- ChangeLog | 12 ++++++++++++ runtime/conf.c | 2 +- tools/omfile.c | 6 +++--- tools/omfwd.c | 2 +- tools/syslogd.c | 13 +++++++------ 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/ChangeLog b/ChangeLog index e88cfbff..4d18623e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,17 @@ --------------------------------------------------------------------------- Version 3.22.1 [v3-stable] (rgerhards), 2009-04-?? +- bugfix: invalid error message issued if $inlcudeConfig was on an empty + set of files (e.g. *.conf, where none such files existed) + thanks to Michael Biebl for reporting this bug +- bugfix: when run in foreground (but not in debug mode), a + debug message ("DoDie called") was emitted at shutdown. Removed. + thanks to Michael Biebl for reporting this bug +- bugfix: some garbagge was emitted to stderr on shutdown. This + garbage consisted of file names, which were written during + startup (key point: not a pointer error) + thanks to Michael Biebl for reporting this bug +- bugfix: startup and shutdown message were emitted to stdout + thanks to Michael Biebl for reporting this bug - bugfix: error messages were not emitted to stderr in forked mode (stderr and stdo are now kept open across forks) - bugfix: internal messages were emitted to whatever file had fd2 when diff --git a/runtime/conf.c b/runtime/conf.c index 510927a2..ad2f4c15 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -206,7 +206,7 @@ doIncludeLine(uchar **pp, __attribute__((unused)) void* pVal) * Required by doIncludeDirectory(). */ result = glob(pattern, GLOB_MARK, NULL, &cfgFiles); - if(result != 0) { + if(result == GLOB_NOSPACE || result == GLOB_ABORTED) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); errmsg.LogError(0, RS_RET_FILE_NOT_FOUND, "error accessing config file or directory '%s': %s", diff --git a/tools/omfile.c b/tools/omfile.c index 5e0f849b..fb83632a 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -128,7 +128,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - printf("[dynamic]\n\ttemplate='%s'" + dbgprintf("[dynamic]\n\ttemplate='%s'" "\tfile cache size=%d\n" "\tcreate directories: %s\n" "\tfile owner %d, group %d\n" @@ -142,9 +142,9 @@ CODESTARTdbgPrintInstInfo pData->bFailOnChown ? "yes" : "no" ); } else { /* regular file */ - printf("%s", pData->f_fname); + dbgprintf("%s", pData->f_fname); if (pData->fd == -1) - printf(" (unused)"); + dbgprintf(" (unused)"); } ENDdbgPrintInstInfo diff --git a/tools/omfwd.c b/tools/omfwd.c index 1dd184ef..e62f84b7 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -177,7 +177,7 @@ ENDfreeInstance BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - printf("%s", pData->f_hname); + dbgprintf("%s", pData->f_hname); ENDdbgPrintInstInfo diff --git a/tools/syslogd.c b/tools/syslogd.c index 1bbf1904..6f32b262 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -903,7 +903,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) MsgSetRcvFrom(pMsg, (char*)glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); /* check if we have an error code associated and, if so, - * adjust the tag. -- r5gerhards, 2008-06-27 + * adjust the tag. -- rgerhards, 2008-06-27 */ if(iErr == NO_ERRCODE) { MsgSetTAG(pMsg, "rsyslogd:"); @@ -925,8 +925,9 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) * permits us to process unmodified config files which otherwise contain a * supressor statement. */ - if(bErrMsgToStderr || iConfigVerify) { - fprintf(stderr, "rsyslogd: %s\n", msg); + if(((Debug || NoFork) && bErrMsgToStderr) || iConfigVerify) { + if(LOG_PRI(pri) == LOG_ERR) + fprintf(stderr, "rsyslogd: %s\n", msg); } if(bHaveMainQueue == 0) { /* not yet in queued mode */ @@ -1882,10 +1883,10 @@ static void doDie(int sig) # define MSG1 "DoDie called.\n" # define MSG2 "DoDie called 5 times - unconditional exit\n" static int iRetries = 0; /* debug aid */ - if(Debug || NoFork) + if(Debug) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { - if(Debug || NoFork) + if(Debug) write(1, MSG2, sizeof(MSG2) - 1); abort(); } @@ -2442,7 +2443,7 @@ init(void) */ snprintf(bufStartUpMsg, sizeof(bufStartUpMsg)/sizeof(char), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ - "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] restart", + "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] (re)start", (int) myPid); logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)bufStartUpMsg, 0); -- cgit From 8628312396b1535c41124e499d292f4d1e77d955 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 13:22:21 +0200 Subject: cleaned up/optimized raw message handling in msg object --- action.c | 2 +- plugins/imdiag/imdiag.c | 3 +- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- plugins/imudp/imudp.c | 5 +--- runtime/msg.c | 43 ++++++++++++++++++++------- runtime/msg.h | 7 ++++- runtime/parser.c | 79 ++++++++++++++++--------------------------------- tcps_sess.c | 6 ++-- tests/nettester.c | 2 +- tests/parsertest.sh | 6 ++-- tools/syslogd.c | 4 +-- 12 files changed, 77 insertions(+), 84 deletions(-) diff --git a/action.c b/action.c index 8bdb6dec..08cdd6fd 100644 --- a/action.c +++ b/action.c @@ -675,7 +675,7 @@ actionWriteToAction(action_t *pAction) datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); MsgSetMSG(pMsg, (char*)szRepMsg); - MsgSetRawMsg(pMsg, (char*)szRepMsg); + MsgSetRawMsgWOSize(pMsg, (char*)szRepMsg); pMsgSave = pAction->f_pMsg; /* save message pointer for later restoration */ pAction->f_pMsg = pMsg; /* use the new msg (pointer will be restored below) */ diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 51f319ca..bfb4a2e5 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -205,8 +205,7 @@ doInjectMsg(int iNum) datetime.getCurrTime(&stTime, &ttGenTime); /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); - CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg)); - pMsg->iLenRawMsg = ustrlen(szMsg); + MsgSetRawMsg(pMsg, (char*) szMsg, ustrlen(szMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 86270e2d..bdd222c4 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -97,7 +97,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); - MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); + MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, (char*)pInfo->pszTag); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 420ebbf1..24f15510 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -96,7 +96,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); - MsgSetRawMsg(pMsg, (char*)msg); + MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetMSG(pMsg, (char*)msg); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 97e66e8e..2340aac4 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -241,10 +241,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, } /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); - /* first trim the buffer to what we have actually received */ - CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); - memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); - pMsg->iLenRawMsg = lenRcvBuf; + MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp"), sizeof("imudp")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; diff --git a/runtime/msg.c b/runtime/msg.c index d9ff2e73..be62e52f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -510,7 +510,8 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - free(pThis->pszRawMsg); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); free(pThis->pszTAG); free(pThis->pszHOSTNAME); free(pThis->pszInputName); @@ -876,11 +877,12 @@ char *getMSG(msg_t *pM) { if(pM == NULL) return ""; - else + else { if(pM->pszMSG == NULL) return ""; else return (char*)pM->pszMSG; + } } @@ -1674,19 +1676,38 @@ void MsgSetMSG(msg_t *pMsg, char* pszMSG) dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); } -/* rgerhards 2004-11-11: set RawMsg in msg object +/* set raw message in message object. Size of message is provided. + * rgerhards, 2009-06-16 */ -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) +void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg) { assert(pMsg != NULL); - if(pMsg->pszRawMsg != NULL) + if(pMsg->pszRawMsg != pMsg->szRawMsg) free(pMsg->pszRawMsg); - pMsg->iLenRawMsg = strlen(pszRawMsg); - if((pMsg->pszRawMsg = (uchar*) malloc(pMsg->iLenRawMsg + 1)) != NULL) - memcpy(pMsg->pszRawMsg, pszRawMsg, pMsg->iLenRawMsg + 1); - else - dbgprintf("Could not allocate memory for pszRawMsg buffer."); + pMsg->iLenRawMsg = lenMsg; + if(pMsg->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pMsg->pszRawMsg = pMsg->szRawMsg; + } else if((pMsg->pszRawMsg = (uchar*) malloc(pMsg->iLenRawMsg + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pMsg->pszRawMsg = pMsg->szRawMsg; + pMsg->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; + } + + memcpy(pMsg->pszRawMsg, pszRawMsg, pMsg->iLenRawMsg); + pMsg->pszRawMsg[pMsg->iLenRawMsg] = '\0'; /* this also works with truncation! */ +} + + +/* set raw message in message object. Size of message is not provided. This + * function should only be used when it is unavoidable (and over time we should + * try to remove it altogether). + * rgerhards, 2009-06-16 + */ +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg) +{ + MsgSetRawMsg(pMsg, pszRawMsg, strlen(pszRawMsg)); } @@ -2558,7 +2579,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("msgFlags")) { pThis->msgFlags = pProp->val.num; } else if(isProp("pszRawMsg")) { - MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); /* enable this, if someone actually uses UxTradMsg, delete after some time has * passed and nobody complained -- rgerhards, 2009-06-16 } else if(isProp("offAfterPRI")) { diff --git a/runtime/msg.h b/runtime/msg.h index 703fdd9f..f2701780 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -28,6 +28,9 @@ #ifndef MSG_H_INCLUDED #define MSG_H_INCLUDED 1 +/* some configuration constants */ +#define CONF_RAWMSG_BUFSIZE 101 + #include #include "obj.h" #include "syslogd-types.h" @@ -109,6 +112,7 @@ struct msg { int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ char pszTimestamp3164[16]; char pszTimestamp3339[33]; }; @@ -149,7 +153,8 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSG(msg_t *pMsg, char* pszMSG); -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, unsigned short *pbMustBeFreed); diff --git a/runtime/parser.c b/runtime/parser.c index c13edb8f..75cde343 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -114,10 +114,8 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) "Message ignored.", ret); FINALIZE; /* unconditional exit, nothing left to do... */ } - free(pMsg->pszRawMsg); - pMsg->pszRawMsg = deflateBuf; - pMsg->iLenRawMsg = iLenDefBuf; - deflateBuf = NULL; /* logically "freed" - caller is now responsible */ + MsgSetRawMsg(pMsg, (char*)deflateBuf, iLenDefBuf); + free(deflateBuf); } finalize_it: if(deflateBuf != NULL) @@ -165,6 +163,8 @@ sanitizeMessage(msg_t *pMsg) size_t iSrc; size_t iDst; size_t iMaxLine; + size_t maxDest; + uchar szSanBuf[32*1024]; /* buffer used for sanitizing a string */ assert(pMsg != NULL); @@ -205,70 +205,43 @@ sanitizeMessage(msg_t *pMsg) } } } - if(bNeedSanitize == 0) { - /* what a shame - we do not have a \0 byte... - * TODO: think about adding it or otherwise be able to use it... - */ - uchar *pRaw; - CHKmalloc(pRaw = realloc(pMsg->pszRawMsg, pMsg->iLenRawMsg + 1)); - pRaw[pMsg->iLenRawMsg] = '\0'; - pMsg->pszRawMsg = pRaw; + + if(!bNeedSanitize) FINALIZE; - } /* now copy over the message and sanitize it */ - /* TODO: can we get cheaper memory alloc? {alloca()?}*/ iMaxLine = glbl.GetMaxLine(); - CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + maxDest = lenMsg * 4; /* message can grow at most four-fold */ + if(maxDest > iMaxLine) + maxDest = iMaxLine; /* but not more than the max size! */ + if(maxDest < sizeof(szSanBuf)) + pDst = szSanBuf; + else + CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; - while(iSrc < lenMsg && iDst < iMaxLine) { + while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ if(pszMsg[iSrc] == '\0') { /* guard against \0 characters... */ - /* changed to the sequence (somewhat) proposed in - * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 - */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ - pDst[iDst++] = cCCEscapeChar; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - } /* if we do not have space, we simply ignore the '\0'... */ - /* log an error? Very questionable... rgerhards, 2006-11-30 */ - /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ - } else if(bEscapeCCOnRcv && iscntrl((int) pszMsg[iSrc])) { + } else if(iscntrl((int) pszMsg[iSrc])) { + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { /* we are configured to escape control characters. Please note * that this most probably break non-western character sets like * Japanese, Korean or Chinese. rgerhards, 2007-07-17 - * Note: sysklogd logs octal values only for DEL and CCs above 127. - * For others, it logs ^n where n is the control char converted to an - * alphabet character. We like consistency and thus escape it to octal - * in all cases. If someone complains, we may change the mode. At least - * we known now what's going on. - * rgerhards, 2007-07-17 */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ - pDst[iDst++] = cCCEscapeChar; - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); - } /* again, if we do not have space, we ignore the char - see comment at '\0' */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); + } } else { pDst[iDst++] = pszMsg[iSrc]; } ++iSrc; } - pDst[iDst] = '\0'; /* space *is* reserved for this! */ - - /* we have a sanitized string. Let's save it now */ - free(pMsg->pszRawMsg); - if((pMsg->pszRawMsg = malloc((iDst+1) * sizeof(uchar))) == NULL) { - /* when we get no new buffer, we use what we already have ;) */ - pMsg->pszRawMsg = pDst; - } else { - /* trim buffer */ - memcpy(pMsg->pszRawMsg, pDst, iDst+1); - free(pDst); /* too big! */ - pMsg->iLenRawMsg = iDst; - } + + MsgSetRawMsg(pMsg, (char*)pDst, iDst); /* save sanitized string */ + + if(pDst != szSanBuf) + free(pDst); finalize_it: RETiRet; diff --git a/tcps_sess.c b/tcps_sess.c index 9353571c..e0bec949 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -234,10 +234,8 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); - /* first trim the buffer to what we have actually received */ - CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); - memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); - pMsg->iLenRawMsg = pThis->iMsg; +dbgprintf("defaultDoSubmit, iMsg %d\n", pThis->iMsg); + MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName, pThis->pLstnInfo->lenInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; diff --git a/tests/nettester.c b/tests/nettester.c index b7b3f892..e5e6278c 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -191,7 +191,7 @@ int openPipe(char *configFile, pid_t *pid, int *pfd) char *newenviron[] = { NULL }; /* debug aide... char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout", - "RSYSLOG_DEBUGLOG=tmp", NULL }; + "RSYSLOG_DEBUGLOG=log", NULL }; */ diff --git a/tests/parsertest.sh b/tests/parsertest.sh index afdb9469..8e04b95e 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,13 +1,13 @@ echo test parsertest via udp $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason -./nettester -tparse1 -iudp +echo test parsertest via tcp +./nettester -tparse1 -itcp if [ "$?" -ne "0" ]; then exit 1 fi -echo test parsertest via tcp -./nettester -tparse1 -itcp +./nettester -tparse1 -iudp if [ "$?" -ne "0" ]; then exit 1 fi diff --git a/tools/syslogd.c b/tools/syslogd.c index 034111f5..91918d96 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -582,7 +582,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f if(pszInputName != NULL) MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName)); MsgSetFlowControlType(pMsg, flowCtlType); - MsgSetRawMsg(pMsg, (char*)msg); + MsgSetRawMsgWOSize(pMsg, (char*)msg); /* test for special codes */ pri = DEFUPRI; @@ -882,7 +882,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) CHKiRet(msgConstruct(&pMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); - MsgSetRawMsg(pMsg, (char*)msg); + MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); -- cgit From 2f86678c577ee469852ffae35123c4a90b12d214 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 14:30:21 +0200 Subject: optimized TAG handling in msg object --- plugins/imfile/imfile.c | 4 +- plugins/imklog/imklog.c | 2 +- runtime/msg.c | 117 +++++++++++++++++++++++++++++------------------- runtime/msg.h | 8 +++- tools/syslogd.c | 6 +-- 5 files changed, 85 insertions(+), 52 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index bdd222c4..0dabc757 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -61,6 +61,7 @@ DEFobjCurrIf(strm) typedef struct fileInfo_s { uchar *pszFileName; uchar *pszTag; + size_t lenTag; uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; @@ -100,7 +101,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); - MsgSetTAG(pMsg, (char*)pInfo->pszTag); + MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); pMsg->bParseHOSTNAME = 0; @@ -471,6 +472,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } else { pThis->pszTag = (uchar*) strdup((char*) pszFileTag); + pThis->lenTag = ustrlen(pThis->pszTag); } if(pszStateFile == NULL) { diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 24f15510..bf60ebe5 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -101,7 +101,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); - MsgSetTAG(pMsg, (char*)pszTag); + MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); pMsg->iFacility = LOG_FAC(iFacility); pMsg->iSeverity = LOG_PRI(iSeverity); pMsg->bParseHOSTNAME = 0; diff --git a/runtime/msg.c b/runtime/msg.c index be62e52f..2df04b2a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -497,6 +497,16 @@ finalize_it: } +/* some free handlers for (slightly) complicated cases... All of them may be called + * with an empty element. + */ +static inline void freeTAG(msg_t *pThis) +{ + if(pThis->iLenTAG >= CONF_TAG_BUFSIZE) + free(pThis->TAG.pszTAG); +} + + BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ int currRefCount; CODESTARTobjDestruct(msg) @@ -512,7 +522,7 @@ CODESTARTobjDestruct(msg) /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ if(pThis->pszRawMsg != pThis->szRawMsg) free(pThis->pszRawMsg); - free(pThis->pszTAG); + freeTAG(pThis); free(pThis->pszHOSTNAME); free(pThis->pszInputName); free(pThis->pszRcvFrom); @@ -602,9 +612,19 @@ msg_t* MsgDup(msg_t* pOld) * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; */ + if(pOld->iLenTAG > 0) { + if(pOld->iLenTAG < CONF_TAG_BUFSIZE) { + memcpy(pNew->TAG.szBuf, pOld->TAG.szBuf, pOld->iLenTAG); + } else { + if((pNew->TAG.pszTAG = srUtilStrDup(pOld->TAG.pszTAG, pOld->iLenTAG)) == NULL) { + msgDestruct(&pNew); + return NULL; + } + pNew->iLenTAG = pOld->iLenTAG; + } + } tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); - tmpCOPYSZ(TAG); tmpCOPYSZ(HOSTNAME); tmpCOPYSZ(RcvFrom); @@ -659,7 +679,10 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszMSG, PSZ); - objSerializePTR(pStrm, pszTAG, PSZ); + + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszTAG"), PROPTYPE_PSZ, (void*) + ((pThis->iLenTAG < CONF_TAG_BUFSIZE) ? pThis->TAG.szBuf : pThis->TAG.pszTAG))); + objSerializePTR(pStrm, pszHOSTNAME, PSZ); objSerializePTR(pStrm, pszInputName, PSZ); objSerializePTR(pStrm, pszRcvFrom, PSZ); @@ -712,18 +735,22 @@ msg_t *MsgAddRef(msg_t *pM) static rsRetVal aquirePROCIDFromTAG(msg_t *pM) { register int i; + uchar *pszTag; DEFiRet; assert(pM != NULL); + if(pM->pCSPROCID != NULL) return RS_RET_OK; /* we are already done ;) */ if(getProtocolVersion(pM) != 0) return RS_RET_OK; /* we can only emulate if we have legacy format */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); + /* find first '['... */ i = 0; - while((i < pM->iLenTAG) && (pM->pszTAG[i] != '[')) + while((i < pM->iLenTAG) && (pszTag[i] != '[')) ++i; if(!(i < pM->iLenTAG)) return RS_RET_OK; /* no [, so can not emulate... */ @@ -733,8 +760,8 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) /* now obtain the PROCID string... */ CHKiRet(cstrConstruct(&pM->pCSPROCID)); rsCStrSetAllocIncrement(pM->pCSPROCID, 16); - while((i < pM->iLenTAG) && (pM->pszTAG[i] != ']')) { - CHKiRet(cstrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); + while((i < pM->iLenTAG) && (pszTag[i] != ']')) { + CHKiRet(cstrAppendChar(pM->pCSPROCID, pszTag[i])); ++i; } @@ -774,22 +801,24 @@ finalize_it: */ static rsRetVal aquireProgramName(msg_t *pM) { - DEFiRet; register int i; + uchar *pszTag; + DEFiRet; assert(pM != NULL); if(pM->pCSProgName == NULL) { /* ok, we do not yet have it. So let's parse the TAG * to obtain it. */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); CHKiRet(cstrConstruct(&pM->pCSProgName)); rsCStrSetAllocIncrement(pM->pCSProgName, 33); for( i = 0 - ; (i < pM->iLenTAG) && isprint((int) pM->pszTAG[i]) - && (pM->pszTAG[i] != '\0') && (pM->pszTAG[i] != ':') - && (pM->pszTAG[i] != '[') && (pM->pszTAG[i] != '/') + ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) + && (pszTag[i] != '\0') && (pszTag[i] != ':') + && (pszTag[i] != '[') && (pszTag[i] != '/') ; ++i) { - CHKiRet(cstrAppendChar(pM->pCSProgName, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSProgName, pszTag[i])); } CHKiRet(cstrFinalize(pM->pCSProgName)); } @@ -808,13 +837,16 @@ finalize_it: * This is especially important as this can be a very common case, e.g. * when BSD syslog is acting as a sender. * rgerhards, 2005-11-10. + * NOTE ********* 2009-06-18 / rgerhards ************* + * This function is being obsoleted by the new handling. I keep it for + * a while, and for oversize tags it is somewhat less optimal than in previous + * versions. This should only happen very seldom. */ void moveHOSTNAMEtoTAG(msg_t *pM) { assert(pM != NULL); - free(pM->pszTAG); - pM->pszTAG = pM->pszHOSTNAME; - pM->iLenTAG = pM->iLenHOSTNAME; + MsgSetTAG(pM, pM->pszHOSTNAME, pM->iLenHOSTNAME); + free(pM->pszHOSTNAME); pM->pszHOSTNAME = NULL; pM->iLenHOSTNAME = 0; } @@ -1264,8 +1296,8 @@ static inline char *getMSGID(msg_t *pM) void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) { assert(pMsg != NULL); - pMsg->iLenTAG = (pBuf == NULL) ? 0 : strlen((char*)pBuf); - pMsg->pszTAG = (uchar*) pBuf; + MsgSetTAG(pMsg, pBuf, ustrlen(pBuf)); + free(pBuf); } @@ -1278,17 +1310,28 @@ void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) } -/* rgerhards 2004-11-16: set TAG in msg object +/* set TAG in msg object + * (rewritten 2009-06-18 rgerhards) */ -void MsgSetTAG(msg_t *pMsg, char* pszTAG) +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) { + uchar *pBuf; assert(pMsg != NULL); - free(pMsg->pszTAG); - pMsg->iLenTAG = strlen(pszTAG); - if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) - memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); - else - dbgprintf("Could not allocate memory in MsgSetTAG()\n"); + + freeTAG(pMsg); + + pMsg->iLenTAG = lenBuf; + if(pMsg->iLenTAG < CONF_RAWMSG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pBuf = pMsg->TAG.szBuf; + } else if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pBuf = pMsg->TAG.szBuf; + pMsg->iLenTAG = CONF_RAWMSG_BUFSIZE - 1; + } + + memcpy(pBuf, pszBuf, pMsg->iLenTAG); + pBuf[pMsg->iLenTAG] = '\0'; /* this also works with truncation! */ } @@ -1305,13 +1348,13 @@ static void tryEmulateTAG(msg_t *pM) uchar *pBuf; assert(pM != NULL); - if(pM->pszTAG != NULL) + if(pM->iLenTAG > 0) return; /* done, no need to emulate */ if(getProtocolVersion(pM) == 1) { if(!strcmp(getPROCID(pM), "-")) { /* no process ID, use APP-NAME only */ - MsgSetTAG(pM, getAPPNAME(pM)); + MsgSetTAG(pM, (uchar*) getAPPNAME(pM), getAPPNAMELen(pM)); } else { /* now we can try to emulate */ iTAGLen = getAPPNAMELen(pM) + getPROCIDLen(pM) + 3; @@ -1324,22 +1367,6 @@ static void tryEmulateTAG(msg_t *pM) } -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getTAGLen(msg_t *pM) -{ - if(pM == NULL) - return 0; - else { - tryEmulateTAG(pM); - if(pM->pszTAG == NULL) - return 0; - else - return pM->iLenTAG; - } -} -#endif - - static inline char *getTAG(msg_t *pM) { char *ret; @@ -1349,10 +1376,10 @@ static inline char *getTAG(msg_t *pM) else { MsgLock(pM); tryEmulateTAG(pM); - if(pM->pszTAG == NULL) + if(pM->iLenTAG == 0) ret = ""; else - ret = (char*) pM->pszTAG; + ret = (char*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); MsgUnlock(pM); } return(ret); @@ -2590,7 +2617,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszUxTradMsg")) { /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { - MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetTAG(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); } else if(isProp("pszInputName")) { MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszRcvFromIP")) { diff --git a/runtime/msg.h b/runtime/msg.h index f2701780..13c2c4ea 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -30,6 +30,7 @@ /* some configuration constants */ #define CONF_RAWMSG_BUFSIZE 101 +#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ #include #include "obj.h" @@ -74,7 +75,6 @@ struct msg { short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ int iLenHOSTNAME; /* Length of HOSTNAME */ @@ -113,6 +113,10 @@ struct msg { ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ + union { + uchar *pszTAG; /* pointer to tag value */ + uchar szBuf[CONF_TAG_BUFSIZE]; + } TAG; char pszTimestamp3164[16]; char pszTimestamp3339[33]; }; @@ -143,7 +147,7 @@ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); -void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); diff --git a/tools/syslogd.c b/tools/syslogd.c index 91918d96..8e804d8f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -890,11 +890,11 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) * adjust the tag. -- rgerhards, 2008-06-27 */ if(iErr == NO_ERRCODE) { - MsgSetTAG(pMsg, "rsyslogd:"); + MsgSetTAG(pMsg, UCHAR_CONSTANT("rsyslogd:"), sizeof("rsyslogd:") - 1); } else { - snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr); + size_t len = snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr); pszTag[32] = '\0'; /* just to make sure... */ - MsgSetTAG(pMsg, (char*)pszTag); + MsgSetTAG(pMsg, pszTag, len); } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); -- cgit From f18c0ffb9a6de737d2b86b3df164ead22ac5ef58 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 15:57:16 +0200 Subject: some optimization in the property replacer --- runtime/msg.c | 88 ++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 2df04b2a..39f2370c 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -876,11 +876,6 @@ char *getProtocolVersionString(msg_t *pM) return(pM->iProtocolVersion ? "1" : "0"); } -int getMSGLen(msg_t *pM) -{ - return((pM == NULL) ? 0 : pM->iLenMSG); -} - static char *getRawMsg(msg_t *pM) { @@ -905,6 +900,12 @@ char *getUxTradMsg(msg_t *pM) } */ + +int getMSGLen(msg_t *pM) +{ + return((pM == NULL) ? 0 : pM->iLenMSG); +} + char *getMSG(msg_t *pM) { if(pM == NULL) @@ -1851,6 +1852,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, { uchar *pName; char *pRes; /* result pointer */ + int bufLen = -1; /* length of string or -1, if not known */ char *pBufStart; char *pBuf; int iLen; @@ -1877,6 +1879,14 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, * property names. These come after || in the ifs below. */ if(!strcmp((char*) pName, "msg")) { pRes = getMSG(pMsg); + bufLen = getMSGLen(pMsg); + } else if(!strcmp((char*) pName, "timestamp") + || !strcmp((char*) pName, "timereported")) { + pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); + } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { + pRes = getHOSTNAME(pMsg); + } else if(!strcmp((char*) pName, "syslogtag")) { + pRes = getTAG(pMsg); } else if(!strcmp((char*) pName, "rawmsg")) { pRes = getRawMsg(pMsg); /* enable this, if someone actually uses UxTradMsg, delete after some time has @@ -1890,10 +1900,6 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (char*) getRcvFrom(pMsg); } else if(!strcmp((char*) pName, "fromhost-ip")) { pRes = (char*) getRcvFromIP(pMsg); - } else if(!strcmp((char*) pName, "source") || !strcmp((char*) pName, "hostname")) { - pRes = getHOSTNAME(pMsg); - } else if(!strcmp((char*) pName, "syslogtag")) { - pRes = getTAG(pMsg); } else if(!strcmp((char*) pName, "pri")) { pRes = getPRI(pMsg); } else if(!strcmp((char*) pName, "pri-text")) { @@ -1917,9 +1923,6 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = getSeverityStr(pMsg); } else if(!strcmp((char*) pName, "timegenerated")) { pRes = getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "timereported") - || !strcmp((char*) pName, "timestamp")) { - pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); } else if(!strcmp((char*) pName, "programname")) { pRes = getProgramName(pMsg); } else if(!strcmp((char*) pName, "protocol-version")) { @@ -2042,6 +2045,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } /* now copy */ memcpy(pBuf, pFld, iLen); + bufLen = iLen; pBuf[iLen] = '\0'; /* terminate it */ if(*pbMustBeFreed == 1) free(pRes); @@ -2086,6 +2090,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } } /* OK, we are at the begin - now let's copy... */ + bufLen = iLen; while(*pSb && iLen) { *pBuf++ = *pSb; ++pSb; @@ -2181,6 +2186,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* Lets copy the matched substring to the buffer */ memcpy(pB, pRes + iOffs + pmatch[pTpe->data.field.iSubMatchToUse].rm_so, iLenBuf); + bufLen = iLenBuf - 1; pB[iLenBuf] = '\0';/* terminate string, did not happen before */ if (*pbMustBeFreed == 1) @@ -2206,28 +2212,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* now check if we need to do our "SP if first char is non-space" hack logic */ if(*pRes && pTpe->data.field.options.bSPIffNo1stSP) { - char *pB; - uchar cFirst = *pRes; - /* here, we always destruct the buffer and return a new one */ - pB = (char *) malloc(2 * sizeof(char)); - if(pB == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } - pRes = pB; - *pbMustBeFreed = 1; - - if(cFirst == ' ') { - /* if we have a SP, we must return an empty string */ - *pRes = '\0'; /* empty */ - } else { - /* if it is no SP, we need to return one */ - *pRes = ' '; - *(pRes+1) = '\0'; - } + uchar cFirst = *pRes; /* save first char */ + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (cFirst == ' ') ? "" : " "; + bufLen = (cFirst == ' ') ? 0 : 1; + *pbMustBeFreed = 0; } if(*pRes) { @@ -2236,11 +2227,12 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.eCaseConv != tplCaseConvNo) { /* we need to obtain a private copy */ - int iBufLen = strlen(pRes); + if(bufLen == -1) + bufLen = strlen(pRes); char *pBStart; char *pB; char *pSrc; - pBStart = pB = malloc((iBufLen + 1) * sizeof(char)); + pBStart = pB = malloc((bufLen + 1) * sizeof(char)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2302,6 +2294,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = iLenBuf; *pbMustBeFreed = 1; } } else if(pTpe->data.field.options.bSpaceCC) { @@ -2320,7 +2313,9 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst = ' '; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = strlen(pRes); + pDst = pDstStart = malloc(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2381,6 +2376,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } } @@ -2420,6 +2416,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = -1; /* TODO: can we do better? */ *pbMustBeFreed = 1; } } else { @@ -2438,7 +2435,9 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst++ = '_'; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = strlen(pRes); + pDst = pDstStart = malloc(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2476,16 +2475,20 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = "_"; + bufLen = 1; *pbMustBeFreed = 0; } } /* Now drop last LF if present (pls note that this must not be done - * if bEscapeCC was set! + * if bEscapeCC was set)! */ if(pTpe->data.field.options.bDropLastLF && !pTpe->data.field.options.bEscapeCC) { - int iLn = strlen(pRes); + int iLn; char *pB; + if(bufLen == -1) + bufLen = strlen(pRes); + iLn = bufLen; if(iLn > 0 && *(pRes + iLn - 1) == '\n') { /* we have a LF! */ /* check if we need to obtain a private copy */ @@ -2501,6 +2504,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pbMustBeFreed = 1; } *(pRes + iLn - 1) = '\0'; /* drop LF ;) */ + --bufLen; } } @@ -2511,10 +2515,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.options.bCSV) { /* we need to obtain a private copy, as we need to at least add the double quotes */ - int iBufLen = strlen(pRes); + int iBufLen; char *pBStart; char *pDst; char *pSrc; + if(bufLen == -1) + bufLen = strlen(pRes); + iBufLen = bufLen; /* the malloc may be optimized, we currently use the worst case... */ pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(char)); if(pDst == NULL) { @@ -2535,6 +2542,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } -- cgit From df9012f755a305ef48037f10fcc9413406894e66 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 16:20:06 +0200 Subject: slight optimization of template generation --- runtime/msg.c | 10 ++++++++-- runtime/msg.h | 2 +- runtime/rule.c | 4 +++- template.c | 6 +++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 39f2370c..67aaf250 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1848,7 +1848,8 @@ static uchar *getNOW(eNOWType eNow) * rgerhards 2005-09-15 */ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed) + cstr_t *pCSPropName, size_t *pPropLen, + unsigned short *pbMustBeFreed) { uchar *pName; char *pRes; /* result pointer */ @@ -2546,6 +2547,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pbMustBeFreed = 1; } + if(bufLen == -1) + bufLen = strlen(pRes); + *pPropLen = bufLen; + /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ return(pRes); } @@ -2561,6 +2566,7 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) { DEFiRet; var_t *pVar; + size_t propLen; uchar *pszProp = NULL; cstr_t *pstrProp; unsigned short bMustBeFreed = 0; @@ -2574,7 +2580,7 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) CHKiRet(var.ConstructFinalize(pVar)); /* always call MsgGetProp() without a template specifier */ - pszProp = (uchar*) MsgGetProp(pThis, NULL, pstrPropName, &bMustBeFreed); + pszProp = (uchar*) MsgGetProp(pThis, NULL, pstrPropName, &propLen, &bMustBeFreed); /* now create a string object out of it and hand that over to the var */ CHKiRet(rsCStrConstructFromszStr(&pstrProp, pszProp)); diff --git a/runtime/msg.h b/runtime/msg.h index 13c2c4ea..48c1090e 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -161,7 +161,7 @@ void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed); + cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); diff --git a/runtime/rule.c b/runtime/rule.c index f17c524e..12494543 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -106,6 +106,7 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) unsigned short pbMustBeFreed; char *pszPropVal; int bRet = 0; + size_t propLen; vm_t *pVM = NULL; var_t *pResult = NULL; @@ -177,7 +178,8 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) bRet = (pResult->val.num) ? 1 : 0; } else { assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.pCSPropName, &pbMustBeFreed); + pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.pCSPropName, &propLen, &pbMustBeFreed); + // TODO: optimize, we now have the length of the property! /* Now do the compares (short list currently ;)) */ switch(pRule->f_filterData.prop.operation ) { diff --git a/template.c b/template.c index 6e34bcd8..aacd4dbd 100644 --- a/template.c +++ b/template.c @@ -103,8 +103,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) FINALIZE; } } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); - iLenVal = strlen((char*) pVal); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &iLenVal, &bMustBeFreed); /* we now need to check if we should use SQL option. In this case, * we must go over the generated string and escape '\'' characters. * rgerhards, 2005-09-22: the option values below look somewhat misplaced, @@ -158,6 +157,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) struct templateEntry *pTpe; uchar **pArr; int iArr; + size_t propLen; unsigned short bMustBeFreed; uchar *pVal; @@ -178,7 +178,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) if(pTpe->eEntryType == CONSTANT) { CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &propLen, &bMustBeFreed); if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ pArr[iArr] = pVal; /* ... so we can use it! */ } else { -- cgit From 2de4964affabc1ccf61bc72426a468fc871a54d0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 16:51:40 +0200 Subject: optimized handling of MSG part in msg object WARNING: currently, message repeation processing is disabled, must be reenabled (but prefer to do some other tests first) --- action.c | 3 ++- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- runtime/msg.c | 30 +++++++++++++++++++++--------- runtime/msg.h | 5 +++-- tools/syslogd.c | 7 +++++-- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/action.c b/action.c index 08cdd6fd..4ab5b81f 100644 --- a/action.c +++ b/action.c @@ -674,7 +674,8 @@ actionWriteToAction(action_t *pAction) */ datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); - MsgSetMSG(pMsg, (char*)szRepMsg); +#pragma warn "need fix msg repeationg handling" + //MsgSetMSG(pMsg, (char*)szRepMsg); MsgSetRawMsgWOSize(pMsg, (char*)szRepMsg); pMsgSave = pAction->f_pMsg; /* save message pointer for later restoration */ diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 0dabc757..0f5f49dc 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -99,7 +99,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); - MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); + MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index bf60ebe5..67fdc8a3 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -97,7 +97,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsgWOSize(pMsg, (char*)msg); - MsgSetMSG(pMsg, (char*)msg); + MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); diff --git a/runtime/msg.c b/runtime/msg.c index 67aaf250..507d041e 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -438,6 +438,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; + pM->offMSG = -1; objConstructSetObjInfo(pM); /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ @@ -527,7 +528,6 @@ CODESTARTobjDestruct(msg) free(pThis->pszInputName); free(pThis->pszRcvFrom); free(pThis->pszRcvFromIP); - free(pThis->pszMSG); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_SecFrac); @@ -608,6 +608,7 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + pNew->offMSG = pOld->offMSG; /* enable this, if someone actually uses UxTradMsg, delete after some time has * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; @@ -624,7 +625,6 @@ msg_t* MsgDup(msg_t* pOld) } } tmpCOPYSZ(RawMsg); - tmpCOPYSZ(MSG); tmpCOPYSZ(HOSTNAME); tmpCOPYSZ(RcvFrom); @@ -668,6 +668,7 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, iProtocolVersion, SHORT); objSerializeSCALAR(pStrm, iSeverity, SHORT); objSerializeSCALAR(pStrm, iFacility, SHORT); + objSerializeSCALAR(pStrm, offMSG, SHORT); objSerializeSCALAR(pStrm, msgFlags, INT); objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); @@ -677,12 +678,10 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, offsAfterPRI, SHORT); */ - objSerializePTR(pStrm, pszRawMsg, PSZ); - objSerializePTR(pStrm, pszMSG, PSZ); - CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszTAG"), PROPTYPE_PSZ, (void*) ((pThis->iLenTAG < CONF_TAG_BUFSIZE) ? pThis->TAG.szBuf : pThis->TAG.pszTAG))); + objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); objSerializePTR(pStrm, pszInputName, PSZ); objSerializePTR(pStrm, pszRcvFrom, PSZ); @@ -911,10 +910,10 @@ char *getMSG(msg_t *pM) if(pM == NULL) return ""; else { - if(pM->pszMSG == NULL) + if(pM->offMSG == -1) return ""; else - return (char*)pM->pszMSG; + return (char*)(pM->pszRawMsg + pM->offMSG); } } @@ -1687,6 +1686,16 @@ void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) } +/* rgerhards 2004-11-09: set MSG in msg object + */ +void MsgSetMSGoffs(msg_t *pMsg, short offs) +{ + assert(pMsg != NULL); + + pMsg->iLenMSG = ustrlen(pMsg->pszRawMsg + offs); + pMsg->offMSG = offs; +} +#if 0 /* rgerhards 2004-11-09: set MSG in msg object */ void MsgSetMSG(msg_t *pMsg, char* pszMSG) @@ -1703,6 +1712,7 @@ void MsgSetMSG(msg_t *pMsg, char* pszMSG) else dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); } +#endif /* set raw message in message object. Size of message is provided. * rgerhards, 2009-06-16 @@ -2619,6 +2629,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) pThis->iFacility = pProp->val.num; } else if(isProp("msgFlags")) { pThis->msgFlags = pProp->val.num; + } else if(isProp("offMSG")) { + pThis->offMSG = pProp->val.num; } else if(isProp("pszRawMsg")) { MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); /* enable this, if someone actually uses UxTradMsg, delete after some time has @@ -2626,8 +2638,6 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("offAfterPRI")) { pThis->offAfterPRI = pProp->val.num; */ - } else if(isProp("pszMSG")) { - MsgSetMSG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszUxTradMsg")) { /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { @@ -2654,6 +2664,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) memcpy(&pThis->tRcvdAt, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); } else if(isProp("tTIMESTAMP")) { memcpy(&pThis->tTIMESTAMP, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); + } else if(isProp("pszMSG")) { + dbgprintf("no longer supported property pszMSG silently ignored\n"); } RETiRet; diff --git a/runtime/msg.h b/runtime/msg.h index 48c1090e..4cdb0762 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -73,7 +73,8 @@ struct msg { * need to preserve cryptographic verifiers. */ int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ - uchar *pszMSG; /* the MSG part itself */ + short offMSG; /* offset at which the MSG part starts in pszRawMsg */ + //uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ @@ -156,7 +157,7 @@ rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); -void MsgSetMSG(msg_t *pMsg, char* pszMSG); +void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); diff --git a/tools/syslogd.c b/tools/syslogd.c index 8e804d8f..11f4ebd9 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1146,7 +1146,8 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) } /* MSG */ - MsgSetMSG(pMsg, (char*)p2parse); + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); + //MsgSetMSG(pMsg, (char*)p2parse); free(pBuf); ENDfunc @@ -1339,7 +1340,9 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } /* The rest is the actual MSG */ - MsgSetMSG(pMsg, (char*)p2parse); +dbgprintf("XXX: msg set msg offset %d, str: '%s', prev '%s'\n", p2parse - pMsg->pszRawMsg, pMsg->pszRawMsg + (p2parse - pMsg->pszRawMsg), p2parse); +// MsgSetMSG(pMsg, (char*)p2parse); + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); ENDfunc return 0; /* all ok */ -- cgit From f33dd51c802a8d49839aa73fb9167d8bc31ea912 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 17:48:11 +0200 Subject: fixed abort condition with oversize tags this was a regression I introduced this afternoon --- runtime/msg.c | 33 +++++++++++---------------------- runtime/msg.h | 1 - template.c | 1 + tests/Makefile.am | 1 + tests/testsuites/oversizeTag-1.parse1 | 3 +++ 5 files changed, 16 insertions(+), 23 deletions(-) create mode 100644 tests/testsuites/oversizeTag-1.parse1 diff --git a/runtime/msg.c b/runtime/msg.c index 507d041e..4b7a0ad4 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1321,13 +1321,17 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) freeTAG(pMsg); pMsg->iLenTAG = lenBuf; - if(pMsg->iLenTAG < CONF_RAWMSG_BUFSIZE) { + if(pMsg->iLenTAG < CONF_TAG_BUFSIZE) { /* small enough: use fixed buffer (faster!) */ pBuf = pMsg->TAG.szBuf; - } else if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { - /* truncate message, better than completely loosing it... */ - pBuf = pMsg->TAG.szBuf; - pMsg->iLenTAG = CONF_RAWMSG_BUFSIZE - 1; + } else { + if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pBuf = pMsg->TAG.szBuf; + pMsg->iLenTAG = CONF_TAG_BUFSIZE - 1; + } else { + pMsg->TAG.pszTAG = pBuf; + } } memcpy(pBuf, pszBuf, pMsg->iLenTAG); @@ -1695,24 +1699,7 @@ void MsgSetMSGoffs(msg_t *pMsg, short offs) pMsg->iLenMSG = ustrlen(pMsg->pszRawMsg + offs); pMsg->offMSG = offs; } -#if 0 -/* rgerhards 2004-11-09: set MSG in msg object - */ -void MsgSetMSG(msg_t *pMsg, char* pszMSG) -{ - assert(pMsg != NULL); - assert(pszMSG != NULL); - if(pMsg->pszMSG != NULL) - free(pMsg->pszMSG); - - pMsg->iLenMSG = strlen(pszMSG); - if((pMsg->pszMSG = (uchar*) malloc(pMsg->iLenMSG + 1)) != NULL) - memcpy(pMsg->pszMSG, pszMSG, pMsg->iLenMSG + 1); - else - dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); -} -#endif /* set raw message in message object. Size of message is provided. * rgerhards, 2009-06-16 @@ -1869,6 +1856,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, int iLen; short iOffs; + BEGINfunc #ifdef FEATURE_REGEXP /* Variables necessary for regular expression matching */ size_t nmatch = 10; @@ -2562,6 +2550,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pPropLen = bufLen; /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ + ENDfunc return(pRes); } diff --git a/runtime/msg.h b/runtime/msg.h index 4cdb0762..ec18b29d 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -74,7 +74,6 @@ struct msg { int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ - //uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ diff --git a/template.c b/template.c index aacd4dbd..93d0539c 100644 --- a/template.c +++ b/template.c @@ -571,6 +571,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) cstrFinalize(pStrB); if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) return 1; +// TODO: another optimization: map name to integer id OPT /* Check frompos, if it has an R, then topos should be a regex */ if(*p == ':') { diff --git a/tests/Makefile.am b/tests/Makefile.am index dbaf85f0..0800f667 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -42,6 +42,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.parse1 \ testsuites/2.parse1 \ testsuites/3.parse1 \ + testsuites/oversizeTag-1.parse1 \ testsuites/date1.parse1 \ testsuites/date2.parse1 \ testsuites/date3.parse1 \ diff --git a/tests/testsuites/oversizeTag-1.parse1 b/tests/testsuites/oversizeTag-1.parse1 new file mode 100644 index 00000000..56510c63 --- /dev/null +++ b/tests/testsuites/oversizeTag-1.parse1 @@ -0,0 +1,3 @@ +<38>Mar 27 19:06:53 source_server 0123456780123456780123456780123456789: MSG part +38,auth,info,Mar 27 19:06:53,source_server,0123456780123456780123456780123456789,0123456780123456780123456780123456789:, MSG part +# yet another real-life sample where we had some issues with -- cgit From dc603a3dbf0bc0654830e7830a07816c23c85ce5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 09:58:33 +0200 Subject: optimized rfc3339 timestamp string generation --- runtime/datetime.c | 103 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 30 deletions(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 40ab4e9c..eac55083 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -616,6 +616,18 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) } +static char *const_number_str[100] = + { "00", "01", "02", "00", "04", "05", "06", "07", "08", "09", + "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", + "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", + "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", + "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", + "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", + "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", + "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", + "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", + "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", + }; /** * Format a syslogTimestamp to a RFC3339 timestamp string (as * specified in syslog-protocol). @@ -626,46 +638,77 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) */ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) { - int iRet; - char szTZ[7]; /* buffer for TZ information */ + int iBuf; + int power; + int secfrac; + short digit; + /* the following table of ten powers saves us some computation */ + static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; + + BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - if(iLenBuf < 20) + if(iLenBuf < 33) return(0); /* we NEED at least 20 bytes */ - /* do TZ information first, this is easier to take care of "Z" zone in rfc3339 */ + /* start with fixed parts */ + /* year yyyy */ + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + /* month */ + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + /* day */ + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = 'T'; + /* hour */ + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = '-'; + /* minute */ + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = '-'; + /* second */ + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + + iBuf = 19; /* points to next free entry, now it becomes dynamic! */ + + if(ts->secfracPrecision > 0) { + pBuf[iBuf++] = '.'; + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } + } + if(ts->OffsetMode == 'Z') { - szTZ[0] = 'Z'; - szTZ[1] = '\0'; + pBuf[iBuf++] = 'Z'; } else { - snprintf(szTZ, sizeof(szTZ) / sizeof(char), "%c%2.2d:%2.2d", - ts->OffsetMode, ts->OffsetHour, ts->OffsetMinute); + pBuf[iBuf++] = ts->OffsetMode; + pBuf[iBuf++] = (ts->OffsetHour / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetHour % 10 + '0'; + pBuf[iBuf++] = ':'; + pBuf[iBuf++] = (ts->OffsetMinute / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetMinute % 10 + '0'; } - if(ts->secfracPrecision > 0) - { /* we now need to include fractional seconds. While doing so, we must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual - * formating (mmmmhhh... kind of self-modifying code... ;)). - */ - char szFmtStr[64]; - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), - "%%04d-%%02d-%%02dT%%02d:%%02d:%%02d.%%0%dd%%s", - ts->secfracPrecision); - iRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, ts->secfrac, szTZ); - } - else - iRet = snprintf(pBuf, iLenBuf, - "%4.4d-%2.2d-%2.2dT%2.2d:%2.2d:%2.2d%s", - ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, szTZ); - return(iRet); + pBuf[iBuf] = '\0'; + + ENDfunc + return iBuf; } /** -- cgit From 77c992e2155420702460e835ce2d561cf2d10fcb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 11:30:57 +0200 Subject: fully optimized datetime module and enhanced test suite tests for the various timestamp formats have been added --- ChangeLog | 4 + runtime/datetime.c | 166 ++++++++++++++++++++------------------ tests/Makefile.am | 22 ++++- tests/diag.sh | 6 ++ tests/nettester.c | 70 +++++++++------- tests/testsuites/1.retry.conf | 2 + tests/testsuites/Apr.ts3164 | 3 + tests/testsuites/Aug.ts3164 | 3 + tests/testsuites/Dec.ts3164 | 3 + tests/testsuites/Feb.ts3164 | 3 + tests/testsuites/Jan.ts3164 | 3 + tests/testsuites/Jul.ts3164 | 3 + tests/testsuites/Jun.ts3164 | 3 + tests/testsuites/Mar.ts3164 | 3 + tests/testsuites/May.ts3164 | 3 + tests/testsuites/Nov.ts3164 | 3 + tests/testsuites/Oct.ts3164 | 3 + tests/testsuites/Sep.ts3164 | 3 + tests/testsuites/master.subsecond | 8 ++ tests/testsuites/master.ts3339 | 22 +++++ tests/testsuites/master.tsmysql | 2 + tests/testsuites/master.tspgsql | 2 + tests/testsuites/mon1digit.ts3164 | 3 + tests/testsuites/mon2digit.ts3164 | 3 + tests/testsuites/parse1udp.conf | 9 +++ tests/testsuites/subsecond.conf | 8 ++ tests/testsuites/ts3164.conf | 8 ++ tests/testsuites/ts3339.conf | 8 ++ tests/testsuites/tsmysql.conf | 8 ++ tests/testsuites/tspgsql.conf | 8 ++ tests/timestamp.sh | 13 +++ 31 files changed, 298 insertions(+), 110 deletions(-) create mode 100644 tests/testsuites/1.retry.conf create mode 100644 tests/testsuites/Apr.ts3164 create mode 100644 tests/testsuites/Aug.ts3164 create mode 100644 tests/testsuites/Dec.ts3164 create mode 100644 tests/testsuites/Feb.ts3164 create mode 100644 tests/testsuites/Jan.ts3164 create mode 100644 tests/testsuites/Jul.ts3164 create mode 100644 tests/testsuites/Jun.ts3164 create mode 100644 tests/testsuites/Mar.ts3164 create mode 100644 tests/testsuites/May.ts3164 create mode 100644 tests/testsuites/Nov.ts3164 create mode 100644 tests/testsuites/Oct.ts3164 create mode 100644 tests/testsuites/Sep.ts3164 create mode 100644 tests/testsuites/master.subsecond create mode 100644 tests/testsuites/master.ts3339 create mode 100644 tests/testsuites/master.tsmysql create mode 100644 tests/testsuites/master.tspgsql create mode 100644 tests/testsuites/mon1digit.ts3164 create mode 100644 tests/testsuites/mon2digit.ts3164 create mode 100644 tests/testsuites/parse1udp.conf create mode 100644 tests/testsuites/subsecond.conf create mode 100644 tests/testsuites/ts3164.conf create mode 100644 tests/testsuites/ts3339.conf create mode 100644 tests/testsuites/tsmysql.conf create mode 100644 tests/testsuites/tspgsql.conf create mode 100755 tests/timestamp.sh diff --git a/ChangeLog b/ChangeLog index 330fd56c..36a819a3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- greatly reduced memory requirements of msg object + to around half of the previous demand. This means that more messages can + be stored in core! Due to fewer cache misses, this also means some + performance improvement. - improved config error messages: now contain a copy of the config line that (most likely) caused the error - removed long-obsoleted property UxTradMsg diff --git a/runtime/datetime.c b/runtime/datetime.c index eac55083..114bd27f 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -49,6 +49,8 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +/* the following table of ten powers saves us some computation */ +static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; /* ------------------------------ methods ------------------------------ */ @@ -544,7 +546,7 @@ finalize_it: * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) +int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) { /* currently we do not consider localtime/utc. This may later be * added. If so, I recommend using a property replacer option @@ -553,28 +555,60 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) * rgerhards, 2007-06-26 */ assert(ts != NULL); - assert(pDst != NULL); + assert(pBuf != NULL); - if (iLenDst < 15) /* we need at least 14 bytes - 14 digits for timestamp + '\n' */ + if (iLenDst < 15) /* we need at least 14 bytes */ return(0); - return(snprintf(pDst, iLenDst, "%4.4d%2.2d%2.2d%2.2d%2.2d%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = (ts->month / 10) % 10 + '0'; + pBuf[5] = ts->month % 10 + '0'; + pBuf[6] = (ts->day / 10) % 10 + '0'; + pBuf[7] = ts->day % 10 + '0'; + pBuf[8] = (ts->hour / 10) % 10 + '0'; + pBuf[9] = ts->hour % 10 + '0'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = (ts->second / 10) % 10 + '0'; + pBuf[13] = ts->second % 10 + '0'; + pBuf[14] = '\0'; + return 15; } -int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) +int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) { /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); - assert(pDst != NULL); + assert(pBuf != NULL); - if (iLenDst < 21) /* we need 20 bytes + '\n' */ - return(0); + if (iLenDst < 20) /* we need 20 bytes */ + return(0); - return(snprintf(pDst, iLenDst, "%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = ' '; + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = ':'; + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = ':'; + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + pBuf[19] = '\0'; + return 19; } @@ -589,45 +623,35 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) */ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) { - int lenRet; - char szFmtStr[64]; + int iBuf; + int power; + int secfrac; + short digit; assert(ts != NULL); assert(pBuf != NULL); assert(iLenBuf >= 10); + iBuf = 0; if(ts->secfracPrecision > 0) - { /* We must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual formating. - */ - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), "%%0%dd", ts->secfracPrecision); - lenRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->secfrac); + { + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } } else { - pBuf[0] = '0'; - pBuf[1] = '\0'; - lenRet = 1; + pBuf[iBuf++] = '0'; } + pBuf[iBuf] = '\0'; - return(lenRet); + return iBuf; } -static char *const_number_str[100] = - { "00", "01", "02", "00", "04", "05", "06", "07", "08", "09", - "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", - "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", - "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", - "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", - "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", - "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", - "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", - "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", - "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", - }; /** * Format a syslogTimestamp to a RFC3339 timestamp string (as * specified in syslog-protocol). @@ -643,15 +667,10 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) int secfrac; short digit; - /* the following table of ten powers saves us some computation */ - static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; - BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - - if(iLenBuf < 33) - return(0); /* we NEED at least 20 bytes */ + assert(iLenBuf < 33); /* start with fixed parts */ /* year yyyy */ @@ -671,11 +690,11 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) /* hour */ pBuf[11] = (ts->hour / 10) % 10 + '0'; pBuf[12] = ts->hour % 10 + '0'; - pBuf[13] = '-'; + pBuf[13] = ':'; /* minute */ pBuf[14] = (ts->minute / 10) % 10 + '0'; pBuf[15] = ts->minute % 10 + '0'; - pBuf[16] = '-'; + pBuf[16] = ':'; /* second */ pBuf[17] = (ts->second / 10) % 10 + '0'; pBuf[18] = ts->second % 10 + '0'; @@ -720,44 +739,36 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) */ int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) { - static char* monthNames[13] = {"ERR", "Jan", "Feb", "Mar", - "Apr", "May", "Jun", "Jul", - "Aug", "Sep", "Oct", "Nov", "Dec"}; + static char* monthNames[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + int iDay; assert(ts != NULL); assert(pBuf != NULL); if(iLenBuf < 16) return(0); /* we NEED 16 bytes */ - return(snprintf(pBuf, iLenBuf, "%s %2d %2.2d:%2.2d:%2.2d", - monthNames[ts->month], ts->day, ts->hour, - ts->minute, ts->second - )); -} -/** - * Format a syslogTimestamp to a text format. - * The caller must provide the timestamp as well as a character - * buffer that will receive the resulting string. The function - * returns the size of the timestamp written in bytes (without - * the string termnator). If 0 is returend, an error occured. - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int formatTimestamp(struct syslogTime *ts, char* pBuf, size_t iLenBuf) -{ - assert(ts != NULL); - assert(pBuf != NULL); - - if(ts->timeType == 1) { - return(formatTimestamp3164(ts, pBuf, iLenBuf)); - } + pBuf[0] = monthNames[(ts->month - 1)% 12][0]; + pBuf[1] = monthNames[(ts->month - 1) % 12][1]; + pBuf[2] = monthNames[(ts->month - 1) % 12][2]; + pBuf[3] = ' '; + iDay = (ts->day / 10) % 10; /* we need to write a space if the first digit is 0 */ + pBuf[4] = iDay ? iDay + '0' : ' '; + pBuf[5] = ts->day % 10 + '0'; + pBuf[6] = ' '; + pBuf[7] = (ts->hour / 10) % 10 + '0'; + pBuf[8] = ts->hour % 10 + '0'; + pBuf[9] = ':'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = ':'; + pBuf[13] = (ts->second / 10) % 10 + '0'; + pBuf[14] = ts->second % 10 + '0'; + pBuf[15] = '\0'; + return 16; /* traditional: number of bytes written */ +} - if(ts->timeType == 2) { - return(formatTimestamp3339(ts, pBuf, iLenBuf)); - } - return(0); -} -#endif /* queryInterface function * rgerhards, 2008-03-05 */ @@ -791,7 +802,6 @@ ENDobjQueryInterface(datetime) BEGINAbstractObjClassInit(datetime, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); - ENDObjClassInit(datetime) /* vi:set ai: diff --git a/tests/Makefile.am b/tests/Makefile.am index 0800f667..955ad405 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -10,7 +10,11 @@ TESTS = $(TESTRUNS) cfg.sh \ queue-persist.sh if ENABLE_OMSTDOUT -TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh +TESTS += omod-if-array.sh \ + parsertest.sh \ + timestamp.sh \ + inputname.sh \ + fieldtest.sh endif endif # if ENABLE_TESTBENCH @@ -37,6 +41,22 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ + timestamp.sh \ + testsuites/ts3164.conf \ + testsuites/mon1digit.ts3164 \ + testsuites/mon2digit.ts3164 \ + testsuites/Jan.ts3164 \ + testsuites/Feb.ts3164 \ + testsuites/Mar.ts3164 \ + testsuites/Apr.ts3164 \ + testsuites/May.ts3164 \ + testsuites/Jun.ts3164 \ + testsuites/Jul.ts3164 \ + testsuites/Aug.ts3164 \ + testsuites/Sep.ts3164 \ + testsuites/Oct.ts3164 \ + testsuites/Nov.ts3164 \ + testsuites/Dec.ts3164 \ testsuites/parse1.conf \ testsuites/field1.conf \ testsuites/1.parse1 \ diff --git a/tests/diag.sh b/tests/diag.sh index 1fa8f62a..2a9d0ee3 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -82,5 +82,11 @@ case $1 in exit 1 fi ;; + 'nettester') # perform nettester-based tests + ./nettester -t$2 -i$3 + if [ "$?" -ne "0" ]; then + exit 1 + fi + ;; *) echo "invalid argument" $1 esac diff --git a/tests/nettester.c b/tests/nettester.c index e5e6278c..b8816b7c 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -61,6 +61,9 @@ static int iPort = 12514; /* port which shall be used for sending data */ static char* pszCustomConf = NULL; /* custom config file, use -c conf to specify */ static int verbose = 0; /* verbose output? -v option */ +/* these two are quick hacks... */ +int iFailed = 0; +int iTests = 0; /* provide user-friednly name of input mode */ @@ -247,38 +250,47 @@ processTestFile(int fd, char *pszFileName) /* skip comments at start of file */ - getline(&testdata, &lenLn, fp); while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + /* this is not perfect, but works ;) */ + if(feof(fp)) + break; - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send (we could use function pointers here...) */ - if(inputMode == inputUDP) { - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - } else { - if(tcpSend(testdata, strlen(testdata)) != 0) - return(2); - } + ++iTests; /* increment test count, we now do one! */ - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send (we could use function pointers here...) */ + if(inputMode == inputUDP) { + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + } else { + if(tcpSend(testdata, strlen(testdata)) != 0) + return(2); + } + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + ++iFailed; + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; } free(testdata); @@ -297,8 +309,6 @@ processTestFile(int fd, char *pszFileName) int doTests(int fd, char *files) { - int iFailed = 0; - int iTests = 0; int ret; char *testFile; glob_t testFiles; @@ -313,7 +323,6 @@ doTests(int fd, char *files) if(stat((char*) testFile, &fileInfo) != 0) continue; /* continue with the next file if we can't stat() the file */ - ++iTests; /* all regular files are run through the test logic. Symlinks don't work. */ if(S_ISREG(fileInfo.st_mode)) { /* config file */ if(verbose) printf("processing test case '%s' ... ", testFile); @@ -324,7 +333,6 @@ doTests(int fd, char *files) if(!verbose) printf("test '%s' ", testFile); printf("failed!\n"); - ++iFailed; } } } diff --git a/tests/testsuites/1.retry.conf b/tests/testsuites/1.retry.conf new file mode 100644 index 00000000..c464b19c --- /dev/null +++ b/tests/testsuites/1.retry.conf @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, diff --git a/tests/testsuites/Apr.ts3164 b/tests/testsuites/Apr.ts3164 new file mode 100644 index 00000000..3134f224 --- /dev/null +++ b/tests/testsuites/Apr.ts3164 @@ -0,0 +1,3 @@ +<167>Apr 6 16:57:54 172.20.245.8 TAG: MSG +Apr 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Aug.ts3164 b/tests/testsuites/Aug.ts3164 new file mode 100644 index 00000000..d9a721eb --- /dev/null +++ b/tests/testsuites/Aug.ts3164 @@ -0,0 +1,3 @@ +<167>Aug 6 16:57:54 172.20.245.8 TAG: MSG +Aug 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Dec.ts3164 b/tests/testsuites/Dec.ts3164 new file mode 100644 index 00000000..080ba401 --- /dev/null +++ b/tests/testsuites/Dec.ts3164 @@ -0,0 +1,3 @@ +<167>Dec 6 16:57:54 172.20.245.8 TAG: MSG +Dec 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Feb.ts3164 b/tests/testsuites/Feb.ts3164 new file mode 100644 index 00000000..d1eaaa33 --- /dev/null +++ b/tests/testsuites/Feb.ts3164 @@ -0,0 +1,3 @@ +<167>Feb 6 16:57:54 172.20.245.8 TAG: MSG +Feb 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jan.ts3164 b/tests/testsuites/Jan.ts3164 new file mode 100644 index 00000000..0cb1c8e2 --- /dev/null +++ b/tests/testsuites/Jan.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 6 16:57:54 172.20.245.8 TAG: MSG +Jan 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jul.ts3164 b/tests/testsuites/Jul.ts3164 new file mode 100644 index 00000000..562e1ec4 --- /dev/null +++ b/tests/testsuites/Jul.ts3164 @@ -0,0 +1,3 @@ +<167>Jul 6 16:57:54 172.20.245.8 TAG: MSG +Jul 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jun.ts3164 b/tests/testsuites/Jun.ts3164 new file mode 100644 index 00000000..ede27e0e --- /dev/null +++ b/tests/testsuites/Jun.ts3164 @@ -0,0 +1,3 @@ +<167>Jun 6 16:57:54 172.20.245.8 TAG: MSG +Jun 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Mar.ts3164 b/tests/testsuites/Mar.ts3164 new file mode 100644 index 00000000..55dd5bc2 --- /dev/null +++ b/tests/testsuites/Mar.ts3164 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 TAG: MSG +Mar 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/May.ts3164 b/tests/testsuites/May.ts3164 new file mode 100644 index 00000000..72a5a301 --- /dev/null +++ b/tests/testsuites/May.ts3164 @@ -0,0 +1,3 @@ +<167>May 6 16:57:54 172.20.245.8 TAG: MSG +May 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Nov.ts3164 b/tests/testsuites/Nov.ts3164 new file mode 100644 index 00000000..e8f00e01 --- /dev/null +++ b/tests/testsuites/Nov.ts3164 @@ -0,0 +1,3 @@ +<167>Nov 6 16:57:54 172.20.245.8 TAG: MSG +Nov 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Oct.ts3164 b/tests/testsuites/Oct.ts3164 new file mode 100644 index 00000000..01423fef --- /dev/null +++ b/tests/testsuites/Oct.ts3164 @@ -0,0 +1,3 @@ +<167>Oct 6 16:57:54 172.20.245.8 TAG: MSG +Oct 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Sep.ts3164 b/tests/testsuites/Sep.ts3164 new file mode 100644 index 00000000..6c9e48e0 --- /dev/null +++ b/tests/testsuites/Sep.ts3164 @@ -0,0 +1,3 @@ +<167>Sep 6 16:57:54 172.20.245.8 TAG: MSG +Sep 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/master.subsecond b/tests/testsuites/master.subsecond new file mode 100644 index 00000000..ee924877 --- /dev/null +++ b/tests/testsuites/master.subsecond @@ -0,0 +1,8 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +003 +# full precision +<34>1 2003-01-23T12:34:56.123456Z mymachine.example.com su - ID47 - MSG +123456 +# without +<34>1 2003-01-23T12:34:56Z mymachine.example.com su - ID47 - MSG +0 diff --git a/tests/testsuites/master.ts3339 b/tests/testsuites/master.ts3339 new file mode 100644 index 00000000..b4dd5f39 --- /dev/null +++ b/tests/testsuites/master.ts3339 @@ -0,0 +1,22 @@ +<34>1 2003-11-11T22:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T22:14:15.003Z +# next test +<34>1 2003-01-11T22:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-01-11T22:14:15.003Z +# next test +<34>1 2003-11-01T22:04:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-01T22:04:15.003Z +# next test +<34>1 2003-11-11T02:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T02:14:15.003Z +# next test +<34>1 2003-11-11T22:04:05.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003Z +# next test +<34>1 2003-11-11T22:04:05.003+02:00 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003+02:00 +# next test +<34>1 2003-11-11T22:04:05.003+01:30 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003+01:30 +<34>1 2003-11-11T22:04:05.123456+01:30 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.123456+01:30 diff --git a/tests/testsuites/master.tsmysql b/tests/testsuites/master.tsmysql new file mode 100644 index 00000000..dc6d85be --- /dev/null +++ b/tests/testsuites/master.tsmysql @@ -0,0 +1,2 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +20030123123456 diff --git a/tests/testsuites/master.tspgsql b/tests/testsuites/master.tspgsql new file mode 100644 index 00000000..d7ac19ff --- /dev/null +++ b/tests/testsuites/master.tspgsql @@ -0,0 +1,2 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +2003-01-23 12:34:56 diff --git a/tests/testsuites/mon1digit.ts3164 b/tests/testsuites/mon1digit.ts3164 new file mode 100644 index 00000000..0cb1c8e2 --- /dev/null +++ b/tests/testsuites/mon1digit.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 6 16:57:54 172.20.245.8 TAG: MSG +Jan 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/mon2digit.ts3164 b/tests/testsuites/mon2digit.ts3164 new file mode 100644 index 00000000..9606961c --- /dev/null +++ b/tests/testsuites/mon2digit.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 16 16:57:54 172.20.245.8 TAG: MSG +Jan 16 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/parse1udp.conf b/tests/testsuites/parse1udp.conf new file mode 100644 index 00000000..0fb7d16d --- /dev/null +++ b/tests/testsuites/parse1udp.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 12514 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/subsecond.conf b/tests/testsuites/subsecond.conf new file mode 100644 index 00000000..58c26cc7 --- /dev/null +++ b/tests/testsuites/subsecond.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-subseconds%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/ts3164.conf b/tests/testsuites/ts3164.conf new file mode 100644 index 00000000..7aa6a8ef --- /dev/null +++ b/tests/testsuites/ts3164.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-rfc3164%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/ts3339.conf b/tests/testsuites/ts3339.conf new file mode 100644 index 00000000..df8f23ac --- /dev/null +++ b/tests/testsuites/ts3339.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-rfc3339%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tsmysql.conf b/tests/testsuites/tsmysql.conf new file mode 100644 index 00000000..f97d4b0a --- /dev/null +++ b/tests/testsuites/tsmysql.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-mysql%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tspgsql.conf b/tests/testsuites/tspgsql.conf new file mode 100644 index 00000000..eb18c091 --- /dev/null +++ b/tests/testsuites/tspgsql.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-pgsql%\n" +*.* :omstdout:;fmt diff --git a/tests/timestamp.sh b/tests/timestamp.sh new file mode 100755 index 00000000..7699a4af --- /dev/null +++ b/tests/timestamp.sh @@ -0,0 +1,13 @@ +echo various timestamp tests +source $srcdir/diag.sh init +source $srcdir/diag.sh nettester ts3164 udp +source $srcdir/diag.sh nettester ts3164 tcp +source $srcdir/diag.sh nettester ts3339 udp +source $srcdir/diag.sh nettester ts3339 tcp +source $srcdir/diag.sh nettester tsmysql udp +source $srcdir/diag.sh nettester tsmysql tcp +source $srcdir/diag.sh nettester tspgsql udp +source $srcdir/diag.sh nettester tspgsql tcp +source $srcdir/diag.sh nettester subsecond udp +source $srcdir/diag.sh nettester subsecond tcp +source $srcdir/diag.sh init -- cgit From 11cb1e008bfb932a159cc746c5d435ae9518ed19 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 11:45:21 +0200 Subject: some cleanup --- runtime/datetime.c | 17 ++++++----------- tests/Makefile.am | 8 ++++++++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 114bd27f..30f397ff 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -556,9 +556,7 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) */ assert(ts != NULL); assert(pBuf != NULL); - - if (iLenDst < 15) /* we need at least 14 bytes */ - return(0); + assert(iLenDst < 15); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -581,12 +579,10 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) { - /* see note in formatTimestampToMySQL, applies here as well */ - assert(ts != NULL); - assert(pBuf != NULL); - - if (iLenDst < 20) /* we need 20 bytes */ - return(0); + /* see note in formatTimestampToMySQL, applies here as well */ + assert(ts != NULL); + assert(pBuf != NULL); + assert(iLenDst < 20): pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -745,8 +741,7 @@ int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) assert(ts != NULL); assert(pBuf != NULL); - if(iLenBuf < 16) - return(0); /* we NEED 16 bytes */ + assert(iLenBuf < 16); pBuf[0] = monthNames[(ts->month - 1)% 12][0]; pBuf[1] = monthNames[(ts->month - 1) % 12][1]; diff --git a/tests/Makefile.am b/tests/Makefile.am index 955ad405..4d38f875 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -57,6 +57,14 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/Oct.ts3164 \ testsuites/Nov.ts3164 \ testsuites/Dec.ts3164 \ + testsuites/ts3339.conf \ + testsuites/master.ts3339 \ + testsuites/tsmysql.conf \ + testsuites/master.tsmysql \ + testsuites/tspgsql.conf \ + testsuites/master.tspgsql \ + testsuites/subsecond.conf \ + testsuites/master.subsecond \ testsuites/parse1.conf \ testsuites/field1.conf \ testsuites/1.parse1 \ -- cgit From dd53709083f3671ac711d5d2cb03bfd9c80a7cf3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 11:58:30 +0200 Subject: typo fix --- runtime/datetime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 30f397ff..b5514e7c 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -582,7 +582,7 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst < 20): + assert(iLenDst < 20); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; -- cgit From 7a695d171436fe249770e8256ae48cd4ed86fd30 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 12:03:56 +0200 Subject: removed uniprocessor optimization ... as it was not even optimal on uniprocessors any longer ;) I keep the config directive in, maybe we can utilize it again at some later point in time (questionable). --- runtime/queue.c | 15 --------------- runtime/queue.h | 13 ++++++------- runtime/wti.c | 5 ----- runtime/wti.h | 3 +-- runtime/wtp.c | 9 --------- runtime/wtp.h | 3 +-- tools/syslogd.c | 2 -- 7 files changed, 8 insertions(+), 42 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 46a3a971..1e3b761f 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1295,7 +1295,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -2216,13 +2215,6 @@ finalize_it: d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); - /* the following pthread_yield is experimental, but brought us performance - * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 - * rgerhards, 2008-10-09 - * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 - */ - if(pThis->bOptimizeUniProc) - pthread_yield(); } RETiRet; @@ -2341,13 +2333,6 @@ finalize_it: d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); dbgoprint((obj_t*) pThis, "MultiEnqObj advised worker start\n"); - /* the following pthread_yield is experimental, but brought us performance - * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 - * rgerhards, 2008-10-09 - * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 - */ - if(pThis->bOptimizeUniProc) - pthread_yield(); } RETiRet; diff --git a/runtime/queue.h b/runtime/queue.h index 5bc03254..1d82d8d9 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -58,11 +58,10 @@ typedef struct qWrkThrd_s { typedef struct queue_s { BEGINobjInstance; queueType_t qType; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ - int bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ - int bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ - int bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ - int bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ + bool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ + bool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ + bool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ + bool bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ int iQueueSize; /* Current number of elements in the queue */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ @@ -73,14 +72,14 @@ typedef struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ - int bSyncQueueFiles;/* if working with files, sync them after each write? */ + bool bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ int iFullDlyMrk; /* if the queue is above this mark, FULL_DELAYable message are put on hold */ int iLightDlyMrk; /* if the queue is above this mark, LIGHT_DELAYable message are put on hold */ int iDiscardSeverity;/* messages of this severity above are discarded on too-full queue */ - int bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ + bool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ int toQShutdown; /* timeout for regular queue shutdown in ms */ int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ diff --git a/runtime/wti.c b/runtime/wti.c index 18767ea1..bfe8f863 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -208,7 +208,6 @@ ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_cond_init(&pThis->condExitDone, NULL); pthread_mutex_init(&pThis->mut, NULL); ENDobjConstruct(wti) @@ -377,10 +376,6 @@ wtiWorker(wti_t *pThis) /* process any pending thread requests */ wtpProcessThrdChanges(pWtp); pthread_testcancel(); /* see big comment in function header */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); /* see big comment in function header */ -# endif /* if we have a rate-limiter set for this worker pool, let's call it. Please * keep in mind that the rate-limiter may hold us for an extended period diff --git a/runtime/wti.h b/runtime/wti.h index 6b60b833..72653b15 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -31,14 +31,13 @@ /* the worker thread instance class */ typedef struct wti_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ pthread_t thrdID; /* thread ID */ qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */ pthread_mutex_t mut; - int bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ + bool bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ uchar *pszDbgHdr; /* header string for debug messages */ } wti_t; diff --git a/runtime/wtp.c b/runtime/wtp.c index df39daa3..611b3f25 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -82,7 +82,6 @@ static rsRetVal NotImplementedDummy() { return RS_RET_OK; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_mutex_init(&pThis->mut, NULL); pthread_mutex_init(&pThis->mutThrdShutdwn, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); @@ -509,14 +508,6 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) dbgprintf("%s: started with state %d, num workers now %d\n", wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd); - /* we try to give the starting worker a little boost. It won't help much as we still - * hold the queue's mutex, but at least it has a chance to start on a single-CPU system. - */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); -# endif - /* indicate we just started a worker and would like to see it running */ wtpSetInactivityGuard(pThis, 1, MUTEX_ALREADY_LOCKED); diff --git a/runtime/wtp.h b/runtime/wtp.h index b9cb07c5..1ce171cc 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -52,13 +52,12 @@ typedef enum { /* the worker thread pool (wtp) object */ typedef struct wtp_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ wtpState_t wtpState; int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ struct wti_s **pWrkr;/* array with control structure for the worker thread(s) associated with this wtp */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ - int bInactivityGuard;/* prevents inactivity due to race condition */ + bool bInactivityGuard;/* prevents inactivity due to race condition */ rsRetVal (*pConsumer)(void *); /* user-supplied consumer function for dewtpd messages */ /* synchronization variables */ pthread_mutex_t mutThrdShutdwn; /* mutex to guard thread shutdown processing */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 11f4ebd9..16f08c21 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1340,8 +1340,6 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } /* The rest is the actual MSG */ -dbgprintf("XXX: msg set msg offset %d, str: '%s', prev '%s'\n", p2parse - pMsg->pszRawMsg, pMsg->pszRawMsg + (p2parse - pMsg->pszRawMsg), p2parse); -// MsgSetMSG(pMsg, (char*)p2parse); MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); ENDfunc -- cgit From 7b3dad877e9faf53609a9f0986925a157f9f6e9b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 12:10:38 +0200 Subject: removed pthread_testcancel() as it is no longer necessary we usually stay long enough inside the actions, so there should be no problem with reaching a cancellation point. Actually, if we really need to cancel, the thread is in an output action (otherwise it would have willingly terminated). --- runtime/wti.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/runtime/wti.c b/runtime/wti.c index bfe8f863..f20812f8 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -327,27 +327,6 @@ wtiWorkerCancelCleanup(void *arg) /* generic worker thread framework - * - * Some special comments below, so that they do not clutter the main function code: - * - * On the use of pthread_testcancel(): - * Now make sure we can get canceled - it is not specified if pthread_setcancelstate() is - * a cancellation point in itself. As we run most of the time without cancel enabled, I fear - * we may never get cancelled if we do not create a cancellation point ourselfs. - * - * On the use of pthread_yield(): - * We yield to give the other threads a chance to obtain the mutex. If we do not - * do that, this thread may very well aquire the mutex again before another thread - * has even a chance to run. The reason is that mutex operations are free to be - * implemented in the quickest possible way (and they typically are!). That is, the - * mutex lock/unlock most probably just does an atomic memory swap and does not necessarily - * schedule other threads waiting on the same mutex. That can lead to the same thread - * aquiring the mutex ever and ever again while all others are starving for it. We - * have exactly seen this behaviour when we deliberately introduced a long-running - * test action which basically did a sleep. I understand that with real actions the - * likelihood of this starvation condition is very low - but it could still happen - * and would be very hard to debug. The yield() is a sure fix, its performance overhead - * should be well accepted given the above facts. -- rgerhards, 2008-01-10 */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal @@ -375,7 +354,6 @@ wtiWorker(wti_t *pThis) while(1) { /* loop will be broken below - need to do mutex locks */ /* process any pending thread requests */ wtpProcessThrdChanges(pWtp); - pthread_testcancel(); /* see big comment in function header */ /* if we have a rate-limiter set for this worker pool, let's call it. Please * keep in mind that the rate-limiter may hold us for an extended period -- cgit From cbad28915a847d9ad461d8099e90ad7903d537a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 12:39:06 +0200 Subject: optimized action.c a bit --- action.c | 34 +++++++++++++++------------------- action.h | 11 ++++++----- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/action.c b/action.c index 4ab5b81f..22755e86 100644 --- a/action.c +++ b/action.c @@ -195,6 +195,7 @@ rsRetVal actionDestruct(action_t *pThis) pthread_mutex_destroy(&pThis->mutActExec); d_free(pThis->pszName); d_free(pThis->ppTpl); + d_free(pThis->ppMsgs); d_free(pThis); RETiRet; @@ -432,23 +433,17 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) int iSleepPeriod; int bCallAction; int iCancelStateSave; - uchar **ppMsgs; /* array of message pointers for doAction */ ASSERT(pAction != NULL); - /* create the array for doAction() message pointers */ - if((ppMsgs = calloc(pAction->iNumTpls, sizeof(uchar *))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { switch(pAction->eParamPassing) { case ACT_STRING_PASSING: - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pAction->ppMsgs[i]))); break; case ACT_ARRAY_PASSING: - CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(ppMsgs[i]))); + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pAction->ppMsgs[i]))); break; default:assert(0); /* software bug if this happens! */ } @@ -487,7 +482,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) if(bCallAction) { /* call configured action */ - iRet = pAction->pMod->mod.om.doAction(ppMsgs, pMsg->msgFlags, pAction->pModData); + iRet = pAction->pMod->mod.om.doAction(pAction->ppMsgs, pMsg->msgFlags, pAction->pModData); if(iRet == RS_RET_SUSPENDED) { dbgprintf("Action requested to be suspended, done that.\n"); actionSuspend(pAction, getActNow(pAction)); @@ -506,25 +501,27 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) finalize_it: /* cleanup */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { - if(ppMsgs[i] != NULL) { + if(pAction->ppMsgs[i] != NULL) { switch(pAction->eParamPassing) { case ACT_ARRAY_PASSING: iArr = 0; - while(((char **)ppMsgs[i])[iArr] != NULL) - d_free(((char **)ppMsgs[i])[iArr++]); - d_free(ppMsgs[i]); + while(((char **)pAction->ppMsgs[i])[iArr] != NULL) { + d_free(((char **)pAction->ppMsgs[i])[iArr++]); + ((char **)pAction->ppMsgs[i])[iArr++] = NULL; + } + d_free(pAction->ppMsgs[i]); + pAction->ppMsgs[i] = NULL; break; case ACT_STRING_PASSING: - d_free(ppMsgs[i]); + d_free(pAction->ppMsgs[i]); break; default: assert(0); } } } - d_free(ppMsgs); - msgDestruct(&pMsg); /* we are now finished with the message */ + msgDestruct(&pMsg); /* we are now finished with the message */ RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" @@ -906,9 +903,8 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques */ if(pAction->iNumTpls > 0) { /* we first need to create the template pointer array */ - if((pAction->ppTpl = calloc(pAction->iNumTpls, sizeof(struct template *))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pAction->ppTpl = (struct template *)calloc(pAction->iNumTpls, sizeof(struct template *))); + CHKmalloc(pAction->ppMsgs = (uchar**) malloc(pAction->iNumTpls * sizeof(uchar *))); } for(i = 0 ; i < pAction->iNumTpls ; ++i) { diff --git a/action.h b/action.h index 2a1487a5..a6c02a8b 100644 --- a/action.h +++ b/action.h @@ -43,10 +43,10 @@ struct action_s { time_t tActNow; /* the current time for an action execution. Initially set to -1 and populated on an as-needed basis. This is a performance optimization. */ time_t tLastExec; /* time this action was last executed */ - int bExecWhenPrevSusp;/* execute only when previous action is suspended? */ + bool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ short bEnabled; /* is the related action enabled (1) or disabled (0)? */ - short bSuspended; /* is the related action temporarily suspended? */ + bool bSuspended; /* is the related action temporarily suspended? */ time_t ttResumeRtry; /* when is it time to retry the resume? */ int iResumeInterval;/* resume interval for this action */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ @@ -57,7 +57,7 @@ struct action_s { time_t tLastOccur; /* time last occurence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ - short bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ + bool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ @@ -66,14 +66,15 @@ struct action_s { int iNumTpls; /* number of array entries for template element below */ struct template **ppTpl;/* array of template to use - strings must be passed to doAction * in this order. */ - struct msg* f_pMsg; /* pointer to the message (this will replace the other vars with msg + msg_t *f_pMsg; /* pointer to the message (this will replace the other vars with msg * content later). This is preserved after the message has been * processed - it is also used to detect duplicates. */ qqueue_t *pQueue; /* action queue */ SYNC_OBJ_TOOL; /* required for mutex support */ - uchar *pszName; /* action name (for documentation) */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ + uchar *pszName; /* action name (for documentation) */ + uchar **ppMsgs; /* pointer to action-calling parameters (kept in structure to save alloc() time!) */ }; typedef struct action_s action_t; -- cgit From 953d015f440224321796105464294006838b5302 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 14:25:27 +0200 Subject: switched default to keep in line with traditional syslogd behaviour --- doc/rsyslog_conf_global.html | 7 +++++-- tools/omfile.c | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d58bcac0..332b6282 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -196,13 +196,16 @@ supported in order to be compliant to the upcoming new syslog RFC series. of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.
  • $OMFileIOBufferSize <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)
    • -
  • $OMFileFlushOnTXEnd <[on/off]>, default off, by default, omfile +
  • $OMFileFlushOnTXEnd <[on/off]>, default on. Omfile has the +capability to writes output using a buffered writer. Disk writes are only done when the buffer is full. So if an error happens during that write, data is potentially lost. In cases where this is unacceptable, set $OMFileFlushOnTXEnd to on. Then, data is written at the end of each transaction (for pre-v5 this means after each log message) and the usual error recovery thus can handle write errors without data loss. Note that this option -severely reduces the effect of zip compression.
    • +severely reduces the effect of zip compression and should be switched to off +for that use case. Note that the default -off- is primarily an aid to preserve +the traditional syslogd behaviour.
  • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/tools/omfile.c b/tools/omfile.c index 675d313e..b62d9c57 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -102,7 +102,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ -static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ +static bool bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static uchar *pszTplName = NULL; /* name of the default template to use */ @@ -715,7 +715,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; - bFlushOnTXEnd = 0; + bFlushOnTXEnd = 1; iIOBufSize = IOBUF_DFLT_SIZE; iFlushInterval = FLUSH_INTRVL_DFLT; if(pszTplName != NULL) { -- cgit From 3abf567d2b57014381eda49018a0e2c21fa1b853 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 16:07:17 +0200 Subject: optimized template string generation --- action.c | 103 ++++++++++++++++++++++++++++++++++++---------------- action.h | 1 + runtime/datetime.c | 8 ++-- runtime/stringbuf.c | 3 -- runtime/stringbuf.h | 2 +- template.c | 97 +++++++++++++++++++++++-------------------------- template.h | 2 +- 7 files changed, 125 insertions(+), 91 deletions(-) diff --git a/action.c b/action.c index 22755e86..d214e808 100644 --- a/action.c +++ b/action.c @@ -178,6 +178,7 @@ actionResetQueueParams(void) */ rsRetVal actionDestruct(action_t *pThis) { + int i; DEFiRet; ASSERT(pThis != NULL); @@ -195,7 +196,33 @@ rsRetVal actionDestruct(action_t *pThis) pthread_mutex_destroy(&pThis->mutActExec); d_free(pThis->pszName); d_free(pThis->ppTpl); + + /* message ptr cleanup */ + for(i = 0 ; i < pThis->iNumTpls ; ++i) { + if(pThis->ppMsgs[i] != NULL) { + switch(pThis->eParamPassing) { + case ACT_ARRAY_PASSING: +#if 0 /* later! */ + iArr = 0; + while(((char **)pThis->ppMsgs[i])[iArr] != NULL) { + d_free(((char **)pThis->ppMsgs[i])[iArr++]); + ((char **)pThis->ppMsgs[i])[iArr++] = NULL; + } + d_free(pThis->ppMsgs[i]); + pThis->ppMsgs[i] = NULL; +#endif + break; + case ACT_STRING_PASSING: + d_free(pThis->ppMsgs[i]); + break; + default: + assert(0); + } + } + } d_free(pThis->ppMsgs); + d_free(pThis->lenMsgs); + d_free(pThis); RETiRet; @@ -440,7 +467,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) for(i = 0 ; i < pAction->iNumTpls ; ++i) { switch(pAction->eParamPassing) { case ACT_STRING_PASSING: - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pAction->ppMsgs[i]))); + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pAction->ppMsgs[i]), &(pAction->lenMsgs[i]))); break; case ACT_ARRAY_PASSING: CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pAction->ppMsgs[i]))); @@ -513,7 +540,6 @@ finalize_it: pAction->ppMsgs[i] = NULL; break; case ACT_STRING_PASSING: - d_free(pAction->ppMsgs[i]); break; default: assert(0); @@ -727,32 +753,13 @@ finalize_it: } -/* call the configured action. Does all necessary housekeeping. - * rgerhards, 2007-08-01 - * FYI: currently, this function is only called from the queue - * consumer. So we (conceptually) run detached from the input - * threads (which also means we may run much later than when the - * message was generated). +/* helper to actonCallAction, mostly needed because of this damn + * pthread_cleanup_push() POSIX macro... */ -#pragma GCC diagnostic ignored "-Wempty-body" -rsRetVal -actionCallAction(action_t *pAction, msg_t *pMsg) +static rsRetVal +doActionCallAction(action_t *pAction, msg_t *pMsg) { DEFiRet; - int iCancelStateSave; - - ISOBJ_TYPE_assert(pMsg, msg); - ASSERT(pAction != NULL); - - /* Make sure nodbody else modifies/uses this action object. Right now, this - * is important because of "message repeated n times" processing and potentially - * multiple worker threads. -- rgerhards, 2007-12-11 - */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - LockObj(pAction); - pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); - pthread_setcancelstate(iCancelStateSave, NULL); - /* first, we need to check if this is a disabled * entry. If so, we must not further process it. * rgerhards 2005-09-26 @@ -813,10 +820,43 @@ actionCallAction(action_t *pAction, msg_t *pMsg) } finalize_it: - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - UnlockObj(pAction); - pthread_cleanup_pop(0); /* remove mutex cleanup handler */ - pthread_setcancelstate(iCancelStateSave, NULL); + RETiRet; +} + +/* call the configured action. Does all necessary housekeeping. + * rgerhards, 2007-08-01 + * FYI: currently, this function is only called from the queue + * consumer. So we (conceptually) run detached from the input + * threads (which also means we may run much later than when the + * message was generated). + */ +#pragma GCC diagnostic ignored "-Wempty-body" +rsRetVal +actionCallAction(action_t *pAction, msg_t *pMsg) +{ + DEFiRet; + int iCancelStateSave; + + ISOBJ_TYPE_assert(pMsg, msg); + ASSERT(pAction != NULL); + + /* We need to lock the mutex only for repeated line processing. + * rgerhards, 2009-06-19 + */ + if(pAction->f_ReduceRepeated == 1) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + LockObj(pAction); + pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); + pthread_setcancelstate(iCancelStateSave, NULL); + iRet = doActionCallAction(pAction, pMsg); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + UnlockObj(pAction); + pthread_cleanup_pop(0); /* remove mutex cleanup handler */ + pthread_setcancelstate(iCancelStateSave, NULL); + } else { + iRet = doActionCallAction(pAction, pMsg); + } + RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" @@ -903,8 +943,9 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques */ if(pAction->iNumTpls > 0) { /* we first need to create the template pointer array */ - CHKmalloc(pAction->ppTpl = (struct template *)calloc(pAction->iNumTpls, sizeof(struct template *))); - CHKmalloc(pAction->ppMsgs = (uchar**) malloc(pAction->iNumTpls * sizeof(uchar *))); + CHKmalloc(pAction->ppTpl = (struct template **)calloc(pAction->iNumTpls, sizeof(struct template *))); + CHKmalloc(pAction->ppMsgs = (uchar**) calloc(pAction->iNumTpls, sizeof(uchar *))); + CHKmalloc(pAction->lenMsgs = (size_t*) calloc(pAction->iNumTpls, sizeof(size_t))); } for(i = 0 ; i < pAction->iNumTpls ; ++i) { diff --git a/action.h b/action.h index a6c02a8b..579a1215 100644 --- a/action.h +++ b/action.h @@ -75,6 +75,7 @@ struct action_s { pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ uchar *pszName; /* action name (for documentation) */ uchar **ppMsgs; /* pointer to action-calling parameters (kept in structure to save alloc() time!) */ + size_t *lenMsgs; /* length of message in ppMsgs */ }; typedef struct action_s action_t; diff --git a/runtime/datetime.c b/runtime/datetime.c index b5514e7c..8f4bfa10 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -556,7 +556,7 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst < 15); + assert(iLenDst >= 15); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -582,7 +582,7 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst < 20); + assert(iLenDst >= 20); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -666,7 +666,7 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf < 33); + assert(iLenBuf >= 33); /* start with fixed parts */ /* year yyyy */ @@ -741,7 +741,7 @@ int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf < 16); + assert(iLenBuf >= 16); pBuf[0] = monthNames[(ts->month - 1)% 12][0]; pBuf[1] = monthNames[(ts->month - 1) % 12][1]; diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index e7fa8c41..d3ddf33a 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -461,14 +461,11 @@ cstrFinalize(cstr_t *pThis) DEFiRet; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - assert(pThis->bIsFinalized == 0); - if(pThis->iStrLen > 0) { /* terminate string only if one exists */ CHKiRet(cstrAppendChar(pThis, '\0')); --pThis->iStrLen; /* do NOT count the \0 byte */ } - pThis->bIsFinalized = 1; finalize_it: RETiRet; diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index b4fc3f3c..9d2e7865 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -48,7 +48,7 @@ typedef struct cstr_s size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ unsigned short iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ - bool bIsFinalized; /**< is this object finished and ready for use? (a debug aid, may be removed later TODO 2009-06-16) */ + bool bIsForeignBuf; /**< is pBuf a buffer provided by someone else? */ } cstr_t; diff --git a/template.c b/template.c index 93d0539c..43a54d98 100644 --- a/template.c +++ b/template.c @@ -49,59 +49,61 @@ static struct template *tplLastStatic = NULL; /* last static element of the temp -/* This functions converts a template into a string. It should - * actually be in template.c, but this requires larger re-structuring - * of the code (because all the property-access functions are static - * to this module). I have placed it next to the iov*() functions, as - * it is somewhat similiar in what it does. - * - * The function takes a pointer to a template and a pointer to a msg object. - * It the creates a string based on the template definition. A pointer - * to that string is returned to the caller. The caller MUST FREE that - * pointer when it is no longer needed. If the function fails, NULL - * is returned. - * If memory allocation fails in this function, we silently return - * NULL. The reason is that we can not do anything against it. And - * if we raise an alert, the memory situation might become even - * worse. So we prefer to let the caller deal with it. - * rgerhards, 2007-07-03 +/* helper to tplToString, extends buffer */ +#define ALLOC_INC 128 +static inline rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize) +{ + uchar *pNewBuf; + size_t iNewSize; + DEFiRet; + + iNewSize = (iMinSize / ALLOC_INC + 1) * ALLOC_INC; + CHKmalloc(pNewBuf = (uchar*) realloc(*pBuf, iNewSize)); + *pBuf = pNewBuf; + *pLenBuf = iNewSize; +dbgprintf("extend buf to at least %ld, done %ld\n", iMinSize, iNewSize); + +finalize_it: + RETiRet; +} + + +/* This functions converts a template into a string. * - * rgerhards, 2007-09-05: I changed the interface to use the standard iRet - * "calling sequence". This greatly eases complexity when it comes to handling - * errors in called modules (plus, it is much nicer). + * The function takes a pointer to a template and a pointer to a msg object + * as well as a pointer to an output buffer and its size. Note that the output + * buffer pointer may be NULL, size 0, in which case a new one is allocated. + * The outpub buffer is grown as required. It is the caller's duty to free the + * buffer when it is done. Note that it is advisable to reuse memory, as this + * offers big performance improvements. + * rewritten 2009-06-19 rgerhards */ -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) +rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *pLenBuf) { DEFiRet; struct templateEntry *pTpe; - cstr_t *pCStr; + int iBuf; unsigned short bMustBeFreed; uchar *pVal; size_t iLenVal; assert(pTpl != NULL); assert(pMsg != NULL); - assert(ppSz != NULL); + assert(ppBuf != NULL); + assert(pLenBuf != NULL); /* loop through the template. We obtain one value * and copy it over to our dynamic string buffer. Then, we * free the obtained value (if requested). We continue this * loop until we got hold of all values. */ - CHKiRet(cstrConstruct(&pCStr)); - pTpe = pTpl->pEntryRoot; + iBuf = 0; while(pTpe != NULL) { if(pTpe->eEntryType == CONSTANT) { - CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, - (uchar *) pTpe->data.constant.pConstant, - pTpe->data.constant.iLenConstant) - ) { - dbgprintf("error %d during tplToString()\n", iRet); - /* it does not make sense to continue now */ - cstrDestruct(&pCStr); - FINALIZE; - } + pVal = (uchar*) pTpe->data.constant.pConstant; + iLenVal = pTpe->data.constant.iLenConstant; + bMustBeFreed = 0; } else if(pTpe->eEntryType == FIELD) { pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &iLenVal, &bMustBeFreed); /* we now need to check if we should use SQL option. In this case, @@ -114,30 +116,23 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); else if(pTpl->optFormatForSQL == 2) doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); - /* value extracted, so lets copy */ - CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, (uchar*) pVal, iLenVal)) { - dbgprintf("error %d during tplToString()\n", iRet); - /* it does not make sense to continue now */ - cstrDestruct(&pCStr); - if(bMustBeFreed) - free(pVal); - FINALIZE; - } - if(bMustBeFreed) - free(pVal); } + /* got source, now copy over */ + if(iBuf + iLenVal + 1 >= *pLenBuf) /* we reserve one char for the final \0! */ + CHKiRet(ExtendBuf(ppBuf, pLenBuf, iBuf + iLenVal + 1)); + + memcpy(*ppBuf + iBuf, pVal, iLenVal); + iBuf += iLenVal; + + if(bMustBeFreed) + free(pVal); + pTpe = pTpe->pNext; } - /* we are done with the template, now let's convert the result into a - * "real" (usable) string and discard the helper structures. - */ - CHKiRet(cstrFinalize(pCStr)); - CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pVal, 0)); + (*ppBuf)[iBuf] = '\0'; /* space was reserved above (see copy) */ finalize_it: - *ppSz = (iRet == RS_RET_OK) ? pVal : NULL; - RETiRet; } diff --git a/template.h b/template.h index 9d794f66..73ed0f84 100644 --- a/template.h +++ b/template.h @@ -127,7 +127,7 @@ void tplLastStaticInit(struct template *tpl); * rgerhards, 2007-08-06 */ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz); +rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); -- cgit