From 9b4a225c5993f96466ac568933c6ad9b819c235b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Oct 2012 19:20:19 +0200 Subject: refactor tpl processor so that date is queried once per template Things like $YEAR, $MONTH required a time() call each. --- plugins/ommongodb/ommongodb.c | 10 +++++----- runtime/msg.c | 43 +++++++++++++++++++++---------------------- runtime/msg.h | 2 +- runtime/ruleset.c | 3 ++- template.c | 19 +++++++++++++++---- 5 files changed, 44 insertions(+), 33 deletions(-) diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c index 2c65f275..57ef20b2 100644 --- a/plugins/ommongodb/ommongodb.c +++ b/plugins/ommongodb/ommongodb.c @@ -233,11 +233,11 @@ getDefaultBSON(msg_t *pMsg) gint64 ts_gen, ts_rcv; /* timestamps: generated, received */ int secfrac; - procid = MsgGetProp(pMsg, NULL, PROP_PROGRAMNAME, NULL, &procid_len, &procid_free); - tag = MsgGetProp(pMsg, NULL, PROP_SYSLOGTAG, NULL, &tag_len, &tag_free); - pid = MsgGetProp(pMsg, NULL, PROP_PROCID, NULL, &pid_len, &pid_free); - sys = MsgGetProp(pMsg, NULL, PROP_HOSTNAME, NULL, &sys_len, &sys_free); - msg = MsgGetProp(pMsg, NULL, PROP_MSG, NULL, &msg_len, &msg_free); + procid = MsgGetProp(pMsg, NULL, PROP_PROGRAMNAME, NULL, &procid_len, &procid_free, NULL); + tag = MsgGetProp(pMsg, NULL, PROP_SYSLOGTAG, NULL, &tag_len, &tag_free, NULL); + pid = MsgGetProp(pMsg, NULL, PROP_PROCID, NULL, &pid_len, &pid_free, NULL); + sys = MsgGetProp(pMsg, NULL, PROP_HOSTNAME, NULL, &sys_len, &sys_free, NULL); + msg = MsgGetProp(pMsg, NULL, PROP_MSG, NULL, &msg_len, &msg_free, NULL); // TODO: move to datetime? Refactor in any case! rgerhards, 2012-03-30 ts_gen = (gint64) datetime.syslogTime2time_t(&pMsg->tTIMESTAMP) * 1000; /* ms! */ diff --git a/runtime/msg.c b/runtime/msg.c index d3c814e2..2ffd4900 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2373,40 +2373,38 @@ char *textpri(char *pRes, size_t pResLen, int pri) */ typedef enum ENOWType { NOW_NOW, NOW_YEAR, NOW_MONTH, NOW_DAY, NOW_HOUR, NOW_HHOUR, NOW_QHOUR, NOW_MINUTE } eNOWType; #define tmpBUFSIZE 16 /* size of formatting buffer */ -static uchar *getNOW(eNOWType eNow) +static uchar *getNOW(eNOWType eNow, struct syslogTime *t) { uchar *pBuf; - struct syslogTime t; if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { return NULL; } - datetime.getCurrTime(&t, NULL); switch(eNow) { case NOW_NOW: - snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day); + snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t->year, t->month, t->day); break; case NOW_YEAR: - snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d", t.year); + snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d", t->year); break; case NOW_MONTH: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.month); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->month); break; case NOW_DAY: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.day); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->day); break; case NOW_HOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.hour); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->hour); break; case NOW_HHOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute / 30); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->minute / 30); break; case NOW_QHOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute / 15); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->minute / 15); break; case NOW_MINUTE: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->minute); break; } @@ -2673,7 +2671,7 @@ finalize_it: * Parameter "bMustBeFreed" is set by this function. It tells the * caller whether or not the string returned must be freed by the * caller itself. It is is 0, the caller MUST NOT free it. If it is - * 1, the caller MUST free 1. Handling this wrongly leads to either + * 1, the caller MUST free it. Handling this wrongly leads to either * a memory leak of a program abort (do to double-frees or frees on * the constant memory pool). So be careful to do it right. * rgerhards 2004-11-23 @@ -2690,7 +2688,7 @@ finalize_it: return(UCHAR_CONSTANT("**OUT OF MEMORY**"));} uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, propid_t propid, es_str_t *propName, rs_size_t *pPropLen, - unsigned short *pbMustBeFreed) + unsigned short *pbMustBeFreed, struct syslogTime *ttNow) { uchar *pRes; /* result pointer */ rs_size_t bufLen = -1; /* length of string or -1, if not known */ @@ -2794,49 +2792,50 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (uchar*)getParseSuccess(pMsg); break; case PROP_SYS_NOW: - if((pRes = getNOW(NOW_NOW)) == NULL) { + if((pRes = getNOW(NOW_NOW, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_YEAR: - if((pRes = getNOW(NOW_YEAR)) == NULL) { + if((pRes = getNOW(NOW_YEAR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else +//TODO set pPropLen! *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_MONTH: - if((pRes = getNOW(NOW_MONTH)) == NULL) { + if((pRes = getNOW(NOW_MONTH, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_DAY: - if((pRes = getNOW(NOW_DAY)) == NULL) { + if((pRes = getNOW(NOW_DAY, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_HOUR: - if((pRes = getNOW(NOW_HOUR)) == NULL) { + if((pRes = getNOW(NOW_HOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_HHOUR: - if((pRes = getNOW(NOW_HHOUR)) == NULL) { + if((pRes = getNOW(NOW_HHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_QHOUR: - if((pRes = getNOW(NOW_QHOUR)) == NULL) { + if((pRes = getNOW(NOW_QHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ break; case PROP_SYS_MINUTE: - if((pRes = getNOW(NOW_MINUTE)) == NULL) { + if((pRes = getNOW(NOW_MINUTE, ttNow)) == NULL) { RET_OUT_OF_MEMORY; } else *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ @@ -3544,7 +3543,7 @@ msgGetMsgVarNew(msg_t *pThis, uchar *name) /* always call MsgGetProp() without a template specifier */ /* TODO: optimize propNameToID() call -- rgerhards, 2009-06-26 */ propNameStrToID(name, &propid); - pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed); + pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed, NULL); estr = es_newStrFromCStr((char*)pszProp, propLen); if(bMustBeFreed) diff --git a/runtime/msg.h b/runtime/msg.h index 396e861f..172ae0da 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -173,7 +173,7 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, propid_t propid, es_str_t *propName, - rs_size_t *pPropLen, unsigned short *pbMustBeFreed); + rs_size_t *pPropLen, unsigned short *pbMustBeFreed, struct syslogTime *ttNow); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name); diff --git a/runtime/ruleset.c b/runtime/ruleset.c index bdeb61b7..d2c21424 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -378,7 +378,8 @@ evalPROPFILT(struct cnfstmt *stmt, msg_t *pMsg) goto done; pszPropVal = MsgGetProp(pMsg, NULL, stmt->d.s_propfilt.propID, - stmt->d.s_propfilt.propName, &propLen, &pbMustBeFreed); + stmt->d.s_propfilt.propName, &propLen, + &pbMustBeFreed, NULL); /* Now do the compares (short list currently ;)) */ switch(stmt->d.s_propfilt.operation ) { diff --git a/template.c b/template.c index 8558dd8a..e09f20a6 100644 --- a/template.c +++ b/template.c @@ -45,12 +45,14 @@ #include "strgen.h" #include "rsconf.h" #include "msg.h" +#include "datetime.h" #include "unicode-helper.h" /* static data */ DEFobjCurrIf(obj) DEFobjCurrIf(errmsg) DEFobjCurrIf(strgen) +DEFobjCurrIf(datetime) /* tables for interfacing with the v6 config system */ static struct cnfparamdescr cnfparamdescr[] = { @@ -149,6 +151,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * unsigned short bMustBeFreed = 0; uchar *pVal; rs_size_t iLenVal = 0; + struct syslogTime ttNow; assert(pTpl != NULL); assert(pMsg != NULL); @@ -176,6 +179,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * } /* we have a "regular" template with template entries */ + datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger /* loop through the template. We obtain one value * and copy it over to our dynamic string buffer. Then, we @@ -191,7 +195,8 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * bMustBeFreed = 0; } else if(pTpe->eEntryType == FIELD) { pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, - pTpe->data.field.propName, &iLenVal, &bMustBeFreed); + pTpe->data.field.propName, &iLenVal, + &bMustBeFreed, &ttNow); /* we now need to check if we should use SQL option. In this case, * we must go over the generated string and escape '\'' characters. * rgerhards, 2005-09-22: the option values below look somewhat misplaced, @@ -254,6 +259,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) rs_size_t propLen; unsigned short bMustBeFreed; uchar *pVal; + struct syslogTime ttNow; assert(pTpl != NULL); assert(pMsg != NULL); @@ -273,6 +279,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) FINALIZE; } + datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger /* loop through the template. We obtain one value, create a * private copy (if necessary), add it to the string array * and then on to the next until we have processed everything. @@ -286,7 +293,8 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); } else if(pTpe->eEntryType == FIELD) { pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, - pTpe->data.field.propName, &propLen, &bMustBeFreed); + pTpe->data.field.propName, &propLen, + &bMustBeFreed, &ttNow); if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ pArr[iArr] = pVal; /* ... so we can use it! */ } else { @@ -318,11 +326,12 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) uchar *pVal; struct json_object *json, *jsonf; rsRetVal localRet; + struct syslogTime ttNow; DEFiRet; assert(pTpl != NULL); assert(pMsg != NULL); - assert(json != NULL); + assert(pjson != NULL); if(pTpl->subtree != NULL){ localRet = jsonFind(pMsg, pTpl->subtree, pjson); @@ -336,6 +345,7 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) } json = json_object_new_object(); + datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger for(pTpe = pTpl->pEntryRoot ; pTpe != NULL ; pTpe = pTpe->pNext) { if(pTpe->eEntryType == CONSTANT) { if(pTpe->fieldName == NULL) @@ -357,7 +367,7 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) } else { pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, pTpe->data.field.propName, &propLen, - &bMustBeFreed); + &bMustBeFreed, &ttNow); if(pTpe->data.field.options.bMandatory || propLen > 0) { jsonf = json_object_new_string_len((char*)pVal, propLen); json_object_object_add(json, (char*)pTpe->fieldName, jsonf); @@ -2140,6 +2150,7 @@ rsRetVal templateInit() DEFiRet; CHKiRet(objGetObjInterface(&obj)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(strgen, CORE_COMPONENT)); finalize_it: -- cgit From dc724fcd2e1a8fcd8c2b3c309fa1c1a3f7bc73ea Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Oct 2012 19:34:08 +0200 Subject: optimize: do "template time() call" only once per batch This still needs more optimizing as the call is very often NOT needed - but we need to know about the templates in that case. --- action.c | 13 ++++++++----- template.c | 24 +++++++++--------------- template.h | 6 +++--- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/action.c b/action.c index 9c06f61e..02c82465 100644 --- a/action.c +++ b/action.c @@ -809,7 +809,8 @@ rsRetVal actionDbgPrint(action_t *pThis) /* prepare the calling parameters for doAction() * rgerhards, 2009-05-07 */ -static rsRetVal prepareDoActionParams(action_t *pAction, batch_obj_t *pElem) +static rsRetVal +prepareDoActionParams(action_t *pAction, batch_obj_t *pElem, struct syslogTime *ttNow) { int i; msg_t *pMsg; @@ -825,17 +826,17 @@ static rsRetVal prepareDoActionParams(action_t *pAction, batch_obj_t *pElem) switch(pAction->eParamPassing) { case ACT_STRING_PASSING: CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pElem->staticActStrings[i]), - &pElem->staticLenStrings[i])); + &pElem->staticLenStrings[i], ttNow)); pElem->staticActParams[i] = pElem->staticActStrings[i]; break; case ACT_ARRAY_PASSING: - CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pElem->staticActParams[i]))); + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pElem->staticActParams[i]), ttNow)); break; case ACT_MSG_PASSING: pElem->staticActParams[i] = (void*) pMsg; break; case ACT_JSON_PASSING: - CHKiRet(tplToJSON(pAction->ppTpl[i], pMsg, &json)); + CHKiRet(tplToJSON(pAction->ppTpl[i], pMsg, &json, ttNow)); pElem->staticActParams[i] = (void*) json; break; default:dbgprintf("software bug/error: unknown pAction->eParamPassing %d in prepareDoActionParams\n", @@ -1226,14 +1227,16 @@ prepareBatch(action_t *pAction, batch_t *pBatch, sbool **activeSave, int *bMustR { int i; batch_obj_t *pElem; + struct syslogTime ttNow; DEFiRet; + datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger pBatch->iDoneUpTo = 0; for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { pElem = &(pBatch->pElem[i]); if(batchIsValidElem(pBatch, i)) { pElem->state = BATCH_STATE_RDY; - if(prepareDoActionParams(pAction, pElem) != RS_RET_OK) { + if(prepareDoActionParams(pAction, pElem, &ttNow) != RS_RET_OK) { /* make sure we have our copy of "active" array */ if(!*bMustRestoreActivePtr) { *activeSave = pBatch->active; diff --git a/template.c b/template.c index e09f20a6..c0011e41 100644 --- a/template.c +++ b/template.c @@ -45,14 +45,12 @@ #include "strgen.h" #include "rsconf.h" #include "msg.h" -#include "datetime.h" #include "unicode-helper.h" /* static data */ DEFobjCurrIf(obj) DEFobjCurrIf(errmsg) DEFobjCurrIf(strgen) -DEFobjCurrIf(datetime) /* tables for interfacing with the v6 config system */ static struct cnfparamdescr cnfparamdescr[] = { @@ -143,7 +141,9 @@ finalize_it: * offers big performance improvements. * rewritten 2009-06-19 rgerhards */ -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *pLenBuf) +rsRetVal +tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *pLenBuf, + struct syslogTime *ttNow) { DEFiRet; struct templateEntry *pTpe; @@ -151,7 +151,6 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * unsigned short bMustBeFreed = 0; uchar *pVal; rs_size_t iLenVal = 0; - struct syslogTime ttNow; assert(pTpl != NULL); assert(pMsg != NULL); @@ -179,7 +178,6 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * } /* we have a "regular" template with template entries */ - datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger /* loop through the template. We obtain one value * and copy it over to our dynamic string buffer. Then, we @@ -196,7 +194,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * } else if(pTpe->eEntryType == FIELD) { pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, pTpe->data.field.propName, &iLenVal, - &bMustBeFreed, &ttNow); + &bMustBeFreed, ttNow); /* we now need to check if we should use SQL option. In this case, * we must go over the generated string and escape '\'' characters. * rgerhards, 2005-09-22: the option values below look somewhat misplaced, @@ -250,7 +248,8 @@ finalize_it: * is indicated by a NULL pointer. * rgerhards, 2009-04-03 */ -rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) +rsRetVal +tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr, struct syslogTime *ttNow) { DEFiRet; struct templateEntry *pTpe; @@ -259,7 +258,6 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) rs_size_t propLen; unsigned short bMustBeFreed; uchar *pVal; - struct syslogTime ttNow; assert(pTpl != NULL); assert(pMsg != NULL); @@ -279,7 +277,6 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) FINALIZE; } - datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger /* loop through the template. We obtain one value, create a * private copy (if necessary), add it to the string array * and then on to the next until we have processed everything. @@ -294,7 +291,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) } else if(pTpe->eEntryType == FIELD) { pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, pTpe->data.field.propName, &propLen, - &bMustBeFreed, &ttNow); + &bMustBeFreed, ttNow); if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ pArr[iArr] = pVal; /* ... so we can use it! */ } else { @@ -318,7 +315,7 @@ finalize_it: * rgerhards, 2012-08-29 */ rsRetVal -tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) +tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson, struct syslogTime *ttNow) { struct templateEntry *pTpe; rs_size_t propLen; @@ -326,7 +323,6 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) uchar *pVal; struct json_object *json, *jsonf; rsRetVal localRet; - struct syslogTime ttNow; DEFiRet; assert(pTpl != NULL); @@ -345,7 +341,6 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) } json = json_object_new_object(); - datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger for(pTpe = pTpl->pEntryRoot ; pTpe != NULL ; pTpe = pTpe->pNext) { if(pTpe->eEntryType == CONSTANT) { if(pTpe->fieldName == NULL) @@ -367,7 +362,7 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) } else { pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, pTpe->data.field.propName, &propLen, - &bMustBeFreed, &ttNow); + &bMustBeFreed, ttNow); if(pTpe->data.field.options.bMandatory || propLen > 0) { jsonf = json_object_new_string_len((char*)pVal, propLen); json_object_object_add(json, (char*)pTpe->fieldName, jsonf); @@ -2150,7 +2145,6 @@ rsRetVal templateInit() DEFiRet; CHKiRet(objGetObjInterface(&obj)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); - CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(strgen, CORE_COMPONENT)); finalize_it: diff --git a/template.h b/template.h index 5a35d274..72053331 100644 --- a/template.h +++ b/template.h @@ -147,9 +147,9 @@ rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); * BEFORE msg.h, even if your code file does not actually need it. * rgerhards, 2007-08-06 */ -rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); -rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **); +rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr, struct syslogTime *ttNow); +rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *, struct syslogTime *ttNow); +rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **, struct syslogTime *ttNow); rsRetVal doEscape(uchar **pp, rs_size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); -- cgit From f85690620b74c076de8f26059e5914f72a8bd912 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 11 Oct 2012 09:12:20 +0200 Subject: do "template date call" only when actually needed --- action.c | 23 ++++++++++++++++++++++- action.h | 1 + template.c | 32 ++++++++++++++++++++++++++++++++ template.h | 1 + 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/action.c b/action.c index 02c82465..cf010d01 100644 --- a/action.c +++ b/action.c @@ -1230,7 +1230,10 @@ prepareBatch(action_t *pAction, batch_t *pBatch, sbool **activeSave, int *bMustR struct syslogTime ttNow; DEFiRet; - datetime.getCurrTime(&ttNow, NULL); // TODO: query time only if required 2012-10-10 rger + if(pAction->requiresDateCall) { + datetime.getCurrTime(&ttNow, NULL); + } + pBatch->iDoneUpTo = 0; for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { pElem = &(pBatch->pElem[i]); @@ -1878,6 +1881,23 @@ actionApplyCnfParam(action_t *pAction, struct cnfparamvals *pvals) return RS_RET_OK; } +/* check if the templates used in this action require a date call + * ($NOW family of properties). + */ +static inline int +actionRequiresDateCall(action_t *pAction) +{ + int i; + int r = 0; + + for(i = 0 ; i < pAction->iNumTpls ; ++i) { + if(tplRequiresDateCall(pAction->ppTpl[i])) { + r = 1; + break; + } + } + return r; +} /* add an Action to the current selector @@ -1983,6 +2003,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, pAction->f_ReduceRepeated = 0; } pAction->eState = ACT_STATE_RDY; /* action is enabled */ + pAction->requiresDateCall = actionRequiresDateCall(pAction); if(bSuspended) actionSuspend(pAction, datetime.GetTime(NULL)); /* "good" time call, only during init and unavoidable */ diff --git a/action.h b/action.h index bce36b4c..177fd682 100644 --- a/action.h +++ b/action.h @@ -70,6 +70,7 @@ struct action_s { void *pModData; /* pointer to module data - content is module-specific */ sbool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ + sbool requiresDateCall;/* do we need to do a date call before creating templates? */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ rsRetVal (*submitToActQ)(action_t *, batch_t *);/* function submit message to action queue */ diff --git a/template.c b/template.c index c0011e41..2ef80f5b 100644 --- a/template.c +++ b/template.c @@ -380,6 +380,38 @@ finalize_it: } +/* Check if the template requires a date call (actually a cached + * date structure). This currently is the case for the $NOW family + * of properties. + */ +int +tplRequiresDateCall(struct template *pTpl) +{ + struct templateEntry *pTpe; + int r = 0; + + if(pTpl->subtree != NULL) + goto done; + + for(pTpe = pTpl->pEntryRoot ; pTpe != NULL ; pTpe = pTpe->pNext) { + switch(pTpe->data.field.propid) { + case PROP_SYS_NOW: + case PROP_SYS_YEAR: + case PROP_SYS_MONTH: + case PROP_SYS_DAY: + case PROP_SYS_HOUR: + case PROP_SYS_HHOUR: + case PROP_SYS_QHOUR: + case PROP_SYS_MINUTE: + r = 1; + goto done; + default:break; + } + } +done: return r; +} + + /* Helper to doEscape. This is called if doEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can diff --git a/template.h b/template.h index 72053331..2bb85a58 100644 --- a/template.h +++ b/template.h @@ -142,6 +142,7 @@ void tplDeleteNew(rsconf_t *conf); void tplPrintList(rsconf_t *conf); void tplLastStaticInit(rsconf_t *conf, struct template *tpl); rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); +int tplRequiresDateCall(struct template *pTpl); /* note: if a compiler warning for undefined type tells you to look at this * code line below, the actual cause is that you currently MUST include template.h * BEFORE msg.h, even if your code file does not actually need it. -- cgit From 398704183200adea592bf17059b3c443160efcf6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 11 Oct 2012 09:57:39 +0200 Subject: slightly improve performance for $NOW family of system properties --- runtime/msg.c | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 2ffd4900..473246e6 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2794,51 +2794,66 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, case PROP_SYS_NOW: if((pRes = getNOW(NOW_NOW, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 10; + } break; case PROP_SYS_YEAR: if((pRes = getNOW(NOW_YEAR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else -//TODO set pPropLen! - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 4; + } break; case PROP_SYS_MONTH: if((pRes = getNOW(NOW_MONTH, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 2; + } break; case PROP_SYS_DAY: if((pRes = getNOW(NOW_DAY, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 2; + } break; case PROP_SYS_HOUR: if((pRes = getNOW(NOW_HOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 2; + } break; case PROP_SYS_HHOUR: if((pRes = getNOW(NOW_HHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 2; + } break; case PROP_SYS_QHOUR: if((pRes = getNOW(NOW_QHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 2; + } break; case PROP_SYS_MINUTE: if((pRes = getNOW(NOW_MINUTE, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + *pPropLen = 2; + } break; case PROP_SYS_MYHOSTNAME: pRes = glbl.GetLocalHostName(); -- cgit From 71a14055adfaa2a43cb1cf0c6d3c05acb6ed1ef0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 11 Oct 2012 11:25:25 +0200 Subject: fix: wrong variable was populated in MsgGetProp() problem in this morning's change, never released --- runtime/msg.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 473246e6..09ee59e2 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2796,7 +2796,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 10; + bufLen = 10; } break; case PROP_SYS_YEAR: @@ -2804,7 +2804,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 4; + bufLen = 4; } break; case PROP_SYS_MONTH: @@ -2812,7 +2812,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 2; + bufLen = 2; } break; case PROP_SYS_DAY: @@ -2820,7 +2820,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 2; + bufLen = 2; } break; case PROP_SYS_HOUR: @@ -2828,7 +2828,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 2; + bufLen = 2; } break; case PROP_SYS_HHOUR: @@ -2836,7 +2836,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 2; + bufLen = 2; } break; case PROP_SYS_QHOUR: @@ -2844,7 +2844,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 2; + bufLen = 2; } break; case PROP_SYS_MINUTE: @@ -2852,7 +2852,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; - *pPropLen = 2; + bufLen = 2; } break; case PROP_SYS_MYHOSTNAME: -- cgit From 3421209cd550b9470c218a4cda578b7aad42529f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 11 Oct 2012 11:33:16 +0200 Subject: optimize property replacer: reduce runtime for simple processing --- runtime/msg.c | 6 ++---- template.c | 22 ++++++++++++++++++++-- template.h | 1 + 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 09ee59e2..b79e4756 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2912,7 +2912,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } /* If we did not receive a template pointer, we are already done... */ - if(pTpe == NULL) { + if(pTpe == NULL || !pTpe->bComplexProcessing) { *pPropLen = (bufLen == -1) ? ustrlen(pRes) : bufLen; return pRes; } @@ -3499,9 +3499,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, jsonField(pTpe, &pRes, pbMustBeFreed, &bufLen); } - if(bufLen == -1) - bufLen = ustrlen(pRes); - *pPropLen = bufLen; + *pPropLen = (bufLen == -1) ? ustrlen(pRes) : bufLen; ENDfunc return(pRes); diff --git a/template.c b/template.c index 4feb02fe..986dbfd6 100644 --- a/template.c +++ b/template.c @@ -828,6 +828,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) /* Check frompos, if it has an R, then topos should be a regex */ if(*p == ':') { + pTpe->bComplexProcessing = 1; ++p; /* eat ':' */ #ifdef FEATURE_REGEXP if(*p == 'R') { @@ -1388,6 +1389,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) int fielddelim = 9; /* default is HT (USACSII 9) */ int re_matchToUse = 0; int re_submatchToUse = 0; + int bComplexProcessing = 0; char *re_expr = NULL; struct cnfparamvals *pvals = NULL; enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE; @@ -1413,23 +1415,31 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) free(tmpstr); } else if(!strcmp(pblkProperty.descr[i].name, "droplastlf")) { droplastlf = pvals[i].val.d.n; + bComplexProcessing = 1; } else if(!strcmp(pblkProperty.descr[i].name, "mandatory")) { mandatory = pvals[i].val.d.n; } else if(!strcmp(pblkProperty.descr[i].name, "spifno1stsp")) { spifno1stsp = pvals[i].val.d.n; + bComplexProcessing = 1; } else if(!strcmp(pblkProperty.descr[i].name, "outname")) { outname = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(pblkProperty.descr[i].name, "position.from")) { frompos = pvals[i].val.d.n; + bComplexProcessing = 1; } else if(!strcmp(pblkProperty.descr[i].name, "position.to")) { topos = pvals[i].val.d.n; + bComplexProcessing = 1; } else if(!strcmp(pblkProperty.descr[i].name, "field.number")) { fieldnum = pvals[i].val.d.n; + bComplexProcessing = 1; } else if(!strcmp(pblkProperty.descr[i].name, "field.delimiter")) { fielddelim = pvals[i].val.d.n; + bComplexProcessing = 1; } else if(!strcmp(pblkProperty.descr[i].name, "regex.expression")) { re_expr = es_str2cstr(pvals[i].val.d.estr, NULL); + bComplexProcessing = 1; } else if(!strcmp(pblkProperty.descr[i].name, "regex.type")) { + bComplexProcessing = 1; if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"BRE", sizeof("BRE")-1)) { re_type = TPL_REGEX_BRE; } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"ERE", sizeof("ERE")-1)) { @@ -1442,6 +1452,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) ABORT_FINALIZE(RS_RET_ERR); } } else if(!strcmp(pblkProperty.descr[i].name, "regex.nomatchmode")) { + bComplexProcessing = 1; if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"DFLT", sizeof("DFLT")-1)) { re_nomatchType = TPL_REGEX_NOMATCH_USE_DFLTSTR; } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"BLANK", sizeof("BLANK")-1)) { @@ -1458,10 +1469,13 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) ABORT_FINALIZE(RS_RET_ERR); } } else if(!strcmp(pblkProperty.descr[i].name, "regex.match")) { + bComplexProcessing = 1; re_matchToUse = pvals[i].val.d.n; } else if(!strcmp(pblkProperty.descr[i].name, "regex.submatch")) { + bComplexProcessing = 1; re_submatchToUse = pvals[i].val.d.n; } else if(!strcmp(pblkProperty.descr[i].name, "format")) { + bComplexProcessing = 1; if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"csv", sizeof("csv")-1)) { formatType = F_CSV; } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"json", sizeof("json")-1)) { @@ -1476,6 +1490,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) ABORT_FINALIZE(RS_RET_ERR); } } else if(!strcmp(pblkProperty.descr[i].name, "controlcharacters")) { + bComplexProcessing = 1; if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"escape", sizeof("escape")-1)) { controlchr = CC_ESCAPE; } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"space", sizeof("space")-1)) { @@ -1490,6 +1505,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) ABORT_FINALIZE(RS_RET_ERR); } } else if(!strcmp(pblkProperty.descr[i].name, "securepath")) { + bComplexProcessing = 1; if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"drop", sizeof("drop")-1)) { secpath = SP_DROP; } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"replace", sizeof("replace")-1)) { @@ -1502,6 +1518,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) ABORT_FINALIZE(RS_RET_ERR); } } else if(!strcmp(pblkProperty.descr[i].name, "caseconversion")) { + bComplexProcessing = 1; if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"lower", sizeof("lower")-1)) { caseconv = tplCaseConvLower; } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"upper", sizeof("upper")-1)) { @@ -1620,6 +1637,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) pTpe->fieldName = outname; if(outname != NULL) pTpe->lenFieldName = ustrlen(outname); + pTpe->bComplexProcessing = bComplexProcessing; pTpe->data.field.eDateFormat = datefmt; if(fieldnum != -1) { pTpe->data.field.has_fields = 1; @@ -2154,6 +2172,8 @@ void tplPrintList(rsconf_t *conf) } break; } + if(pTpe->bComplexProcessing) + dbgprintf("[COMPLEX]"); dbgprintf("\n"); pTpe = pTpe->pNext; } @@ -2167,8 +2187,6 @@ int tplGetEntryCount(struct template *pTpl) return(pTpl->tpenElements); } -/* our init function. TODO: remove once converted to a class - */ rsRetVal templateInit() { DEFiRet; diff --git a/template.h b/template.h index 2bb85a58..018e2f52 100644 --- a/template.h +++ b/template.h @@ -72,6 +72,7 @@ struct templateEntry { enum EntryTypes eEntryType; uchar *fieldName; /**< field name to be used for structured output */ int lenFieldName; + sbool bComplexProcessing; /**< set if complex processing (options, etc) is required */ union { struct { uchar *pConstant; /* pointer to constant value */ -- cgit From 3b5e2ec7b2d21cc145b62e8e6bcf5e58eb94918c Mon Sep 17 00:00:00 2001 From: Miloslav Trmač Date: Tue, 25 Sep 2012 13:52:41 +0200 Subject: Fix crash when date properties are used without a template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E.g. in > set $!time = $timereported; > set $!time_rcvd = $timegenerated; pTpe is set to NULL by the caller. (Is "default" the correct format to use?) Signed-off-by: Miloslav Trmač --- runtime/msg.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index b79e4756..b0b93f98 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2696,6 +2696,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, uchar *pBuf; int iLen; short iOffs; + enum tplFormatTypes datefmt; BEGINfunc assert(pMsg != NULL); @@ -2715,7 +2716,11 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, bufLen = getMSGLen(pMsg); break; case PROP_TIMESTAMP: - pRes = (uchar*)getTimeReported(pMsg, pTpe->data.field.eDateFormat); + if (pTpe != NULL) + datefmt = pTpe->data.field.eDateFormat; + else + datefmt = tplFmtDefault; + pRes = (uchar*)getTimeReported(pMsg, datefmt); break; case PROP_HOSTNAME: pRes = (uchar*)getHOSTNAME(pMsg); @@ -2765,7 +2770,11 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (uchar*)getSeverityStr(pMsg); break; case PROP_TIMEGENERATED: - pRes = (uchar*)getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); + if (pTpe != NULL) + datefmt = pTpe->data.field.eDateFormat; + else + datefmt = tplFmtDefault; + pRes = (uchar*)getTimeGenerated(pMsg, datefmt); break; case PROP_PROGRAMNAME: pRes = getProgramName(pMsg, LOCK_MUTEX); -- cgit