From de84a12f8a5f140c0f7b8e00f4cac92ef13cd866 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 29 Jun 2009 16:53:26 +0200 Subject: introduced the idea of detached properties some things inside the message can be used over a large number of messages and need to to be allocated and re-written every time. I now begin to implement this as a "prop_t" object, first use for the inputName. Some input modules are already converted, some others to go. Will do a little performance check on the new method before I go further. Also, this commit has some cleanup and a few bug fixes that prevented compiliation in debug mode (I overlooked this as I did not compile for debug, what I normally do, and the automatted test also does not do that) --- runtime/datetime.c | 5 --- runtime/datetime.h | 2 -- runtime/msg.c | 91 ++++++++++++++++++++++++++++++++++++++--------------- runtime/msg.h | 11 +++++-- runtime/prop.c | 63 +++++++++++++++++++++++++++++++++++++ runtime/prop.h | 11 ++++++- runtime/rsyslog.c | 3 ++ runtime/rsyslog.h | 1 + runtime/stringbuf.h | 2 +- 9 files changed, 152 insertions(+), 37 deletions(-) (limited to 'runtime') diff --git a/runtime/datetime.c b/runtime/datetime.c index e0f3f5fa..ea67eec5 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -556,7 +556,6 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf) */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst >= 15); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -582,7 +581,6 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf) /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst >= 20); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -665,7 +663,6 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf) BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf >= 33); /* start with fixed parts */ /* year yyyy */ @@ -740,8 +737,6 @@ int formatTimestamp3164(struct syslogTime *ts, char* pBuf) assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf >= 16); - pBuf[0] = monthNames[(ts->month - 1)% 12][0]; pBuf[1] = monthNames[(ts->month - 1) % 12][1]; pBuf[2] = monthNames[(ts->month - 1) % 12][2]; diff --git a/runtime/datetime.h b/runtime/datetime.h index 79a86d05..58f368e7 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -23,8 +23,6 @@ #ifndef INCLUDED_DATETIME_H #define INCLUDED_DATETIME_H -#include "datetime.h" - /* TODO: define error codes */ #define NO_ERRCODE -1 diff --git a/runtime/msg.c b/runtime/msg.c index c4ba18df..47b4ed85 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -48,6 +48,7 @@ #include "atomic.h" #include "unicode-helper.h" #include "ruleset.h" +#include "prop.h" /* static data */ DEFobjStaticHelpers @@ -55,6 +56,7 @@ DEFobjCurrIf(var) DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) +DEFobjCurrIf(prop) static struct { uchar *pszName; @@ -273,12 +275,26 @@ static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", /* some forward declarations */ static int getAPPNAMELen(msg_t *pM, bool bLockMutex); + static inline int getProtocolVersion(msg_t *pM) { return(pM->iProtocolVersion); } +static inline void +getInputName(msg_t *pM, uchar **ppsz, int *plen) +{ + BEGINfunc + if(pM == NULL) { + *ppsz = UCHAR_CONSTANT(""); + *plen = 0; + } else { + prop.GetString(pM->pInputName, ppsz, plen); + } + ENDfunc +} + /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we * include them as hooks only. The idea is that we need to guard @@ -431,7 +447,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->iFacility = -1; pM->offAfterPRI = 0; pM->offMSG = -1; - pM->iLenInputName = 0; pM->iProtocolVersion = 0; pM->msgFlags = 0; pM->iLenRawMsg = 0; @@ -444,7 +459,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pszHOSTNAME = NULL; pM->pszRcvFrom = NULL; pM->pszRcvFromIP = NULL; - pM->pszInputName = NULL; pM->pszRcvdAt3164 = NULL; pM->pszRcvdAt3339 = NULL; pM->pszRcvdAt_MySQL = NULL; @@ -458,6 +472,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pCSAPPNAME = NULL; pM->pCSPROCID = NULL; pM->pCSMSGID = NULL; + pM->pInputName = NULL; pM->pRuleset = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); @@ -556,7 +571,8 @@ CODESTARTobjDestruct(msg) free(pThis->pszRawMsg); freeTAG(pThis); freeHOSTNAME(pThis); - free(pThis->pszInputName); + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); free(pThis->pszRcvFrom); free(pThis->pszRcvFromIP); free(pThis->pszRcvdAt3164); @@ -719,11 +735,16 @@ msg_t* MsgDup(msg_t* pOld) */ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) { + uchar *psz; + int len; DEFiRet; assert(pThis != NULL); assert(pStrm != NULL); + /* "pump" some property values into strings */ + + /* then serialize elements */ CHKiRet(obj.BeginSerialize(pStrm, (obj_t*) pThis)); objSerializeSCALAR(pStrm, iProtocolVersion, SHORT); objSerializeSCALAR(pStrm, iSeverity, SHORT); @@ -743,7 +764,9 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); - objSerializePTR(pStrm, pszInputName, PSZ); + getInputName(pThis, &psz, &len); + objSerializeSCALAR_VAR(pStrm, "pszInputName", PSZ, psz); + //objSerializePTR(pStrm, pszInputName, PSZ); objSerializePTR(pStrm, pszRcvFrom, PSZ); objSerializePTR(pStrm, pszRcvFromIP, PSZ); @@ -1427,18 +1450,6 @@ char *getHOSTNAME(msg_t *pM) } -static uchar *getInputName(msg_t *pM) -{ - if(pM == NULL) - return (uchar*) ""; - else - if(pM->pszInputName == NULL) - return (uchar*) ""; - else - return pM->pszInputName; -} - - uchar *getRcvFrom(msg_t *pM) { if(pM == NULL) @@ -1598,14 +1609,31 @@ static int getAPPNAMELen(msg_t *pM, bool bLockMutex) /* rgerhards 2008-09-10: set pszInputName in msg object * rgerhards, 2009-06-16 */ -void MsgSetInputName(msg_t *pMsg, uchar* pszInputName, size_t lenInputName) +void MsgSetInputName(msg_t *pThis, prop_t *inputName) { - assert(pMsg != NULL); - free(pMsg->pszInputName); - pMsg->iLenInputName = lenInputName; - if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { - memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); - } + assert(pThis != NULL); + + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); + pThis->pInputName = inputName; +} + +/* to be removed soon: work-around for those tht can not natively generate an + * input name. + * rgerhards, 2009-06-29 + */ +void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len) +{ + prop_t *pProp; + assert(pThis != NULL); + + /* we need to create a property */ + prop.Construct(&pProp); + prop.SetString(pProp, psz, len); + prop.ConstructFinalize(pProp); + prop.AddRef(pProp); + MsgSetInputName(pThis, pProp); + prop.Destruct(&pProp); } /* rgerhards 2004-11-16: set pszRcvFrom in msg object @@ -1875,6 +1903,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, short iOffs; BEGINfunc +dbgprintf("XXXX: msgGetProp for %d\n", propID); assert(pMsg != NULL); assert(pbMustBeFreed != NULL); @@ -1910,7 +1939,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, break; */ case PROP_INPUTNAME: - pRes = (char*) getInputName(pMsg); +RUNLOG; + getInputName(pMsg, ((uchar**) &pRes), &bufLen); +RUNLOG; +RUNLOG_VAR("%p", pRes); break; case PROP_FROMHOST: pRes = (char*) getRcvFrom(pMsg); @@ -2738,6 +2770,7 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) #define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) { + prop_t *myProp; DEFiRet; ISOBJ_TYPE_assert(pThis, msg); @@ -2765,7 +2798,13 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszTAG")) { MsgSetTAG(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); + /* we need to create a property */ + CHKiRet(prop.Construct(&myProp)); + CHKiRet(prop.SetString(myProp, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr))); + CHKiRet(prop.ConstructFinalize(myProp)); + prop.AddRef(myProp); + MsgSetInputName(pThis, myProp); + prop.Destruct(&myProp); } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { @@ -2790,6 +2829,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) dbgprintf("no longer supported property pszMSG silently ignored\n"); } +finalize_it: RETiRet; } #undef isProp @@ -2833,6 +2873,7 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_SERIALIZE, MsgSerialize); diff --git a/runtime/msg.h b/runtime/msg.h index e6b25e6c..a1fc535b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -72,7 +72,6 @@ struct msg { short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ - short iLenInputName; /* Length of pszInputName */ short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ int msgFlags; /* flags associated with this message */ int iLenRawMsg; /* length of raw message */ @@ -86,7 +85,6 @@ struct msg { uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ uchar *pszRcvFrom; /* System message was received from */ uchar *pszRcvFromIP; /* IP of system message was received from */ - uchar *pszInputName; /* name of the input module that submitted this message */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ @@ -100,6 +98,7 @@ struct msg { cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + prop_t *pInputName; /* input name property */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -144,7 +143,7 @@ rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -void MsgSetInputName(msg_t *pMsg, uchar*, size_t); +void MsgSetInputName(msg_t *pMsg, prop_t*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); @@ -166,6 +165,12 @@ char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); + +// REMOVE: +void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len); + + + /* TODO: remove these five (so far used in action.c) */ char *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); diff --git a/runtime/prop.c b/runtime/prop.c index 02be315f..989657dd 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -36,9 +36,12 @@ #include "config.h" #include #include +#include #include "rsyslog.h" #include "obj.h" +#include "obj-types.h" +#include "atomic.h" #include "prop.h" /* static data */ @@ -48,8 +51,45 @@ DEFobjStaticHelpers /* Standard-Constructor */ BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! */ + pThis->iRefCount = 1; ENDobjConstruct(prop) +/* set string, we make our own private copy! This MUST only be called BEFORE + * ConstructFinalize()! + */ +static rsRetVal SetString(prop_t *pThis, uchar *psz, int len) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + pThis->len = len; + if(len < CONF_PROP_BUFSIZE) { + memcpy(pThis->szVal.sz, psz, len + 1); + } else { + CHKmalloc(pThis->szVal.psz = malloc(len + 1)); + memcpy(pThis->szVal.sz, psz, len + 1); + } + +finalize_it: + RETiRet; +} + + +/* get string */ +static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) +{ + BEGINfunc + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len < CONF_PROP_BUFSIZE) + *ppsz = pThis->szVal.sz; + else + *ppsz = pThis->szVal.psz; + *plen = pThis->len; + ENDfunc + return RS_RET_OK; +} + /* ConstructionFinalizer * rgerhards, 2008-01-09 @@ -63,9 +103,29 @@ propConstructFinalize(prop_t __attribute__((unused)) *pThis) } +/* add a new reference. It is VERY IMPORTANT to call this function whenever + * the property is handed over to some entitiy that later call Destruct() on it. + */ +static rsRetVal AddRef(prop_t *pThis) +{ + ATOMIC_INC(pThis->iRefCount); + return RS_RET_OK; +} + + /* destructor for the prop object */ BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ + int currRefCount; CODESTARTobjDestruct(prop) + currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); + if(currRefCount == 0) { + /* (only) in this case we need to actually destruct the object */ +dbgprintf("XXXXX: propDestruct: ptr %p, pThis %p, len %d\n", pThis->szVal.psz, pThis, pThis->len); + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + } else { + pThis = NULL; /* tell framework NOT to destructing the object! */ + } ENDobjDestruct(prop) @@ -94,6 +154,9 @@ CODESTARTobjQueryInterface(prop) pIf->ConstructFinalize = propConstructFinalize; pIf->Destruct = propDestruct; pIf->DebugPrint = propDebugPrint; + pIf->SetString = SetString; + pIf->GetString = GetString; + pIf->AddRef = AddRef; finalize_it: ENDobjQueryInterface(prop) diff --git a/runtime/prop.h b/runtime/prop.h index 7fc466b5..1d18c650 100644 --- a/runtime/prop.h +++ b/runtime/prop.h @@ -28,14 +28,23 @@ /* the prop object */ struct prop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int iRefCount; /* reference counter */ + union { + uchar *psz; /* stored string */ + uchar sz[CONF_PROP_BUFSIZE]; + } szVal; + int len; /* we use int intentionally, otherwise we may get some troubles... */ }; /* interfaces */ BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ INTERFACEObjDebugPrint(prop); rsRetVal (*Construct)(prop_t **ppThis); - rsRetVal (*ConstructFinalize)(prop_t __attribute__((unused)) *pThis); + rsRetVal (*ConstructFinalize)(prop_t *pThis); rsRetVal (*Destruct)(prop_t **ppThis); + rsRetVal (*SetString)(prop_t *pThis, uchar* psz, int len); + rsRetVal (*GetString)(prop_t *pThis, uchar** ppsz, int *plen); + rsRetVal (*AddRef)(prop_t *pThis); ENDinterface(prop) #define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 6f732f0e..faa74427 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -77,6 +77,7 @@ #include "conf.h" #include "glbl.h" #include "errmsg.h" +#include "prop.h" #include "rule.h" #include "ruleset.h" @@ -150,6 +151,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(glblClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "datetime"; CHKiRet(datetimeClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "prop"; + CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ctok_token"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 4e0f6e6c..8a043dde 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -39,6 +39,7 @@ #define CONF_RAWMSG_BUFSIZE 101 #define CONF_TAG_BUFSIZE 32 #define CONF_HOSTNAME_BUFSIZE 32 +#define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ /* ############################################################# * diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 400c74eb..c5130238 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -35,6 +35,7 @@ #ifndef _STRINGBUF_H_INCLUDED__ #define _STRINGBUF_H_INCLUDED__ 1 +#include /** * The dynamic string buffer object. @@ -48,7 +49,6 @@ typedef struct cstr_s uchar *pszBuf; /**< pointer to the sz version of the string (after it has been created )*/ size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ - bool bIsForeignBuf; /**< is pBuf a buffer provided by someone else? */ } cstr_t; -- cgit