From 7f0cd8c8b93ca395949e5d28c3a8f422e6695c8d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Apr 2012 15:20:16 +0200 Subject: added the "jsonf" property replacer option (and fieldname) & bugfix bugfix: property replacer option "json" could lead to content loss message was truncated if escaping was necessary --- ChangeLog | 3 ++ doc/property_replacer.html | 30 +++++++++-- runtime/msg.c | 130 ++++++++++++++++++++++++++++++++------------- template.c | 73 ++++++++++++++++++------- template.h | 4 +- 5 files changed, 181 insertions(+), 59 deletions(-) diff --git a/ChangeLog b/ChangeLog index 732bb8bb..1532c9db 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,8 @@ --------------------------------------------------------------------------- Version 6.3.9 [DEVEL] 2012-04-?? +- added the "jsonf" property replacer option (and fieldname) +- bugfix: property replacer option "json" could lead to content loss + message was truncated if escaping was necessary - bugfix: assigned ruleset was lost when using disk queues This looked quite hard to diagnose for disk-assisted queues, as the pure memory part worked well, but ruleset info was lost for messages diff --git a/doc/property_replacer.html b/doc/property_replacer.html index 1178b4ca..5ee1d096 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -13,7 +13,7 @@ the value, e.g. by converting all characters to lower case.

Syslog message properties are used inside templates. They are accessed by putting them between percent signs. Properties can be modified by the property replacer. The full syntax is as follows:

-
%propname:fromChar:toChar:options%
+
%propname:fromChar:toChar:options:fieldname%

Available Properties

propname is the name of the property to access. It is case-insensitive (prior to 3.17.0, they were case-senstive). @@ -349,7 +349,24 @@ case-insensitive. Currently, the following options are defined: json encode the value so that it can be used inside a JSON field. This means that several characters (according to the JSON spec) are being escaped, for -example US-ASCII LF is replaced by "\n". +example US-ASCII LF is replaced by "\n". +The json option cannot be used together with either jsonf or csv options. + + + +jsonf +(available in 6.3.9+) +This signifies that the property should be expressed as a json field. +That means not only the property is written, but rather a complete json field in +the format
+"fieldname"="value" +where "filedname" is the assigend field name (or the property name if none was assigned) +and value is the end result of property replacer operation. Note that value supports +all property replacer options, like substrings, case converson and the like. +Values are properly json-escaped. However, field names are (currently) not. It is +expected that proper field names are configured. +The jsonf option cannot be used together with either json or csv options. + csv @@ -360,6 +377,7 @@ text, you need to define a proper template. An example is this one:
$template csvline,"%syslogtag:::csv%,%msg:::csv%"
Most importantly, you need to provide the commas between the fields inside the template. +The csv option cannot be used together with either json or jsonf options.
This feature was introduced in rsyslog 4.1.6. @@ -465,13 +483,19 @@ Useful for secure pathname generation (with dynafiles). them. For example "escape-cc,sp-if-no-1st-sp". If you use conflicting options together, the last one will override the previous one. For example, using "escape-cc,drop-cc" will use drop-cc and "drop-cc,escape-cc" will use escape-cc mode. +

Fieldname

+

(available in 6.3.9+) +

This field permits to specify a field name for structured-data emitting property replacer +options. It was initially introduced to support the "jsonf" option, for which it provides +the capability to set an alternative field name. If it is not specified, it defaults to +the property name.

Further Links

[manual index] [rsyslog.conf] diff --git a/runtime/msg.c b/runtime/msg.c index 1cc5f6b4..9c7a2203 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -263,6 +263,9 @@ static struct { { UCHAR_CONSTANT("190"), 5}, { UCHAR_CONSTANT("191"), 5} }; +static char hexdigit[16] = + {'0', '1', '2', '3', '4', '5', '6', '7', '8', + '9', 'A', 'B', 'C', 'D', 'E', 'F' }; /*syslog facility names (as of RFC5424) */ static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", "syslog", "lpr", @@ -423,7 +426,6 @@ resolveDNS(msg_t *pMsg) { uchar fromHostFQDN[NI_MAXHOST]; DEFiRet; -dbgprintf("XXXX: in msg/resolveDNS (dnscache)\n"); MsgLock(pMsg); CHKiRet(objUse(net, CORE_COMPONENT)); if(pMsg->msgFlags & NEEDS_DNSRESOL) { @@ -2371,81 +2373,67 @@ finalize_it: } -/* encode a property in JSON escaped format. This is a helper - * to MsgGetProp. It needs to update all provided parameters. - * Note: Code is borrowed from libee (my own code, so ASL 2.0 - * is fine with it); this function may later be replaced by - * some "better" and more complete implementation (maybe from - * libee or its helpers). - * For performance reasons, we begin to copy the string only - * when we recognice that we actually need to do some escaping. - * rgerhards, 2012-03-16 +/* Encode a JSON value and add it to provided string. Note that + * the string object may be NULL. In this case, it is created + * if and only if escaping is needed. */ static rsRetVal -jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen) +jsonAddVal(uchar *pSrc, unsigned buflen, es_str_t **dst) { - static char hexdigit[16] = - {'0', '1', '2', '3', '4', '5', '6', '7', '8', - '9', 'A', 'B', 'C', 'D', 'E', 'F' }; unsigned char c; es_size_t i; char numbuf[4]; int j; - unsigned buflen; - uchar *pSrc; - es_str_t *dst = NULL; DEFiRet; - pSrc = *ppRes; - buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen; for(i = 0 ; i < buflen ; ++i) { c = pSrc[i]; if( (c >= 0x23 && c <= 0x5b) || (c >= 0x5d /* && c <= 0x10FFFF*/) || c == 0x20 || c == 0x21) { /* no need to escape */ - if(dst != NULL) - es_addChar(&dst, c); + if(*dst != NULL) + es_addChar(dst, c); } else { - if(dst == NULL) { + if(*dst == NULL) { if(i == 0) { /* we hope we have only few escapes... */ - dst = es_newStr(buflen+10); + *dst = es_newStr(buflen+10); } else { - dst = es_newStrFromBuf((char*)pSrc, i-1); + *dst = es_newStrFromBuf((char*)pSrc, i-1); } - if(dst == NULL) { + if(*dst == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } } /* we must escape, try RFC4627-defined special sequences first */ switch(c) { case '\0': - es_addBuf(&dst, "\\u0000", 6); + es_addBuf(dst, "\\u0000", 6); break; case '\"': - es_addBuf(&dst, "\\\"", 2); + es_addBuf(dst, "\\\"", 2); break; case '/': - es_addBuf(&dst, "\\/", 2); + es_addBuf(dst, "\\/", 2); break; case '\\': - es_addBuf(&dst, "\\\\", 2); + es_addBuf(dst, "\\\\", 2); break; case '\010': - es_addBuf(&dst, "\\b", 2); + es_addBuf(dst, "\\b", 2); break; case '\014': - es_addBuf(&dst, "\\f", 2); + es_addBuf(dst, "\\f", 2); break; case '\n': - es_addBuf(&dst, "\\n", 2); + es_addBuf(dst, "\\n", 2); break; case '\r': - es_addBuf(&dst, "\\r", 2); + es_addBuf(dst, "\\r", 2); break; case '\t': - es_addBuf(&dst, "\\t", 2); + es_addBuf(dst, "\\t", 2); break; default: /* TODO : proper Unicode encoding (see header comment) */ @@ -2453,12 +2441,38 @@ jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen) numbuf[3-j] = hexdigit[c % 16]; c = c / 16; } - es_addBuf(&dst, "\\u", 2); - es_addBuf(&dst, numbuf, 4); + es_addBuf(dst, "\\u", 2); + es_addBuf(dst, numbuf, 4); break; } } } +finalize_it: + RETiRet; +} + + +/* encode a property in JSON escaped format. This is a helper + * to MsgGetProp. It needs to update all provided parameters. + * Note: Code is borrowed from libee (my own code, so ASL 2.0 + * is fine with it); this function may later be replaced by + * some "better" and more complete implementation (maybe from + * libee or its helpers). + * For performance reasons, we begin to copy the string only + * when we recognice that we actually need to do some escaping. + * rgerhards, 2012-03-16 + */ +static rsRetVal +jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen) +{ + unsigned buflen; + uchar *pSrc; + es_str_t *dst = NULL; + DEFiRet; + + pSrc = *ppRes; + buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen; + CHKiRet(jsonAddVal(pSrc, buflen, &dst)); if(dst != NULL) { /* we updated the string and need to replace the @@ -2468,6 +2482,7 @@ jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen) free(*ppRes); *ppRes = (uchar*)es_str2cstr(dst, NULL); *pbMustBeFreed = 1; + *pBufLen = -1; es_deleteStr(dst); } @@ -2476,6 +2491,46 @@ finalize_it: } +/* Format a property as JSON field, that means + * "name"="value" + * where value is JSON-escaped (here we assume that the name + * only contains characters from the valid character set). + * Note: this function duplicates code from jsonEncode(). + * TODO: these two functions should be combined, at least if + * that makes any sense from a performance PoV - definitely + * something to consider at a later stage. rgerhards, 2012-04-19 + */ +static rsRetVal +jsonField(struct templateEntry *pTpe, uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen) +{ + unsigned buflen; + uchar *pSrc; + es_str_t *dst = NULL; + DEFiRet; + + pSrc = *ppRes; + buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen; + /* we hope we have only few escapes... */ + dst = es_newStr(buflen+es_strlen(pTpe->data.field.fieldName)+15); + es_addChar(&dst, '"'); + es_addStr(&dst, pTpe->data.field.fieldName); + es_addBufConstcstr(&dst, "\"=\""); + CHKiRet(jsonAddVal(pSrc, buflen, &dst)); + es_addChar(&dst, '"'); + + if(*pbMustBeFreed) + free(*ppRes); + /* we know we do not have \0 chars - so the size does not change */ + *pBufLen = es_strlen(dst); + *ppRes = (uchar*)es_str2cstr(dst, NULL); + *pbMustBeFreed = 1; + es_deleteStr(dst); + +finalize_it: + RETiRet; +} + + /* This function returns a string-representation of the * requested message property. This is a generic function used * to abstract properties so that these can be easier @@ -3301,6 +3356,8 @@ dbgprintf("prop repl 4, pRes='%s', len %d\n", pRes, bufLen); *pbMustBeFreed = 1; } else if(pTpe->data.field.options.bJSON) { jsonEncode(&pRes, pbMustBeFreed, &bufLen); + } else if(pTpe->data.field.options.bJSONf) { + jsonField(pTpe, &pRes, pbMustBeFreed, &bufLen); } if(bufLen == -1) @@ -3369,7 +3426,6 @@ msgGetMsgVarNew(msg_t *pThis, uchar *name) propNameStrToID(name, &propid); pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed); -dbgprintf("ZZZZ: var %s returns '%s'\n", name, pszProp); estr = es_newStrFromCStr((char*)pszProp, propLen); if(bMustBeFreed) free(pszProp); diff --git a/template.c b/template.c index 2326819c..c6878459 100644 --- a/template.c +++ b/template.c @@ -493,18 +493,18 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) p = *pp; - while(*p && *p != '%') { + while(*p && *p != '%' && *p != ':') { /* outer loop - until end of options */ i = 0; while((i < sizeof(Buf) / sizeof(char)) && - *p && *p != '%' && *p != ',') { + *p && *p != '%' && *p != ':' && *p != ',') { /* inner loop - until end of ONE option */ Buf[i++] = tolower((int)*p); ++p; } Buf[i] = '\0'; /* terminate */ /* check if we need to skip oversize option */ - while(*p && *p != '%' && *p != ',') + while(*p && *p != '%' && *p != ':' && *p != ',') ++p; /* just skip */ if(*p == ',') ++p; /* eat ',' */ @@ -544,19 +544,26 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) } else if(!strcmp((char*)Buf, "secpath-replace")) { pTpe->data.field.options.bSecPathReplace = 1; } else if(!strcmp((char*)Buf, "csv")) { - if(pTpe->data.field.options.bJSON) { - errmsg.LogError(0, NO_ERRCODE, "error: can not specify " - "both csv and json options - csv ignored"); + if(pTpe->data.field.options.bJSON || pTpe->data.field.options.bJSONf) { + errmsg.LogError(0, NO_ERRCODE, "error: can only specify " + "one option out of (json, jsonf, csv) - csv ignored"); } else { pTpe->data.field.options.bCSV = 1; } } else if(!strcmp((char*)Buf, "json")) { - if(pTpe->data.field.options.bCSV) { - errmsg.LogError(0, NO_ERRCODE, "error: can not specify " - "both csv and json options - json ignored"); + if(pTpe->data.field.options.bCSV || pTpe->data.field.options.bJSON) { + errmsg.LogError(0, NO_ERRCODE, "error: can only specify " + "one option out of (json, jsonf, csv) - json ignored"); } else { pTpe->data.field.options.bJSON = 1; } + } else if(!strcmp((char*)Buf, "jsonf")) { + if(pTpe->data.field.options.bCSV || pTpe->data.field.options.bJSON) { + errmsg.LogError(0, NO_ERRCODE, "error: can only specify " + "one option out of (json, jsonf, csv) - jsonf ignored"); + } else { + pTpe->data.field.options.bJSONf = 1; + } } else { dbgprintf("Invalid field option '%s' specified - ignored.\n", Buf); } @@ -573,7 +580,8 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) static int do_Parameter(unsigned char **pp, struct template *pTpl) { unsigned char *p; - cstr_t *pStrB; + cstr_t *pStrProp; + cstr_t *pStrField = NULL; struct templateEntry *pTpe; int iNum; /* to compute numbers */ #ifdef FEATURE_REGEXP @@ -590,7 +598,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) p = (unsigned char*) *pp; - if(cstrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrProp) != RS_RET_OK) return 1; if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -601,25 +609,24 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) pTpe->eEntryType = FIELD; while(*p && *p != '%' && *p != ':') { - cstrAppendChar(pStrB, tolower(*p)); + cstrAppendChar(pStrProp, tolower(*p)); ++p; /* do NOT do this in tolower()! */ } /* got the name */ - cstrFinalize(pStrB); + cstrFinalize(pStrProp); - if(propNameToID(pStrB, &pTpe->data.field.propid) != RS_RET_OK) { - cstrDestruct(&pStrB); + if(propNameToID(pStrProp, &pTpe->data.field.propid) != RS_RET_OK) { + cstrDestruct(&pStrProp); return 1; } if(pTpe->data.field.propid == PROP_CEE) { /* in CEE case, we need to preserve the actual property name */ - if((pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrB)+2, cstrLen(pStrB)-2)) == NULL) { - cstrDestruct(&pStrB); + if((pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp)+2, cstrLen(pStrProp)-2)) == NULL) { + cstrDestruct(&pStrProp); return 1; } } - cstrDestruct(&pStrB); /* Check frompos, if it has an R, then topos should be a regex */ if(*p == ':') { @@ -873,6 +880,34 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) doOptions(&p, pTpe); } + /* check field name */ + if(*p == ':') { + ++p; /* eat ':' */ + if(cstrConstruct(&pStrField) != RS_RET_OK) + return 1; + while(*p != ':' && *p != '%' && *p != '\0') { + cstrAppendChar(pStrField, *p); + ++p; + } + cstrFinalize(pStrField); + } + + /* save field name - if none was given, use the property name instead */ + if(pStrField == NULL) { + if((pTpe->data.field.fieldName = + es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp), cstrLen(pStrProp))) == NULL) { + return 1; + } + } else { + if((pTpe->data.field.fieldName = + es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrField), cstrLen(pStrField))) == NULL) { + return 1; + } + cstrDestruct(&pStrField); + } + + cstrDestruct(&pStrProp); + if(*p) ++p; /* eat '%' */ *pp = p; @@ -1130,6 +1165,8 @@ void tplDeleteAll(rsconf_t *conf) } if(pTpeDel->data.field.propName != NULL) es_deleteStr(pTpeDel->data.field.propName); + if(pTpeDel->data.field.fieldName != NULL) + es_deleteStr(pTpeDel->data.field.fieldName); #endif break; } diff --git a/template.h b/template.h index a9eff6b1..f55f64b3 100644 --- a/template.h +++ b/template.h @@ -91,6 +91,7 @@ struct templateEntry { int field_expand; /* use multiple instances of the field delimiter as a single one? */ es_str_t *propName; /**< property name (currently being used for CEE only) */ + es_str_t *fieldName; /**< field name to be used for structured output */ enum tplFormatTypes eDateFormat; enum tplFormatCaseConvTypes eCaseConv; @@ -101,9 +102,10 @@ struct templateEntry { unsigned bDropLastLF: 1; /* drop last LF char in msg (PIX!) */ unsigned bSecPathDrop: 1; /* drop slashes, replace dots, empty string */ unsigned bSecPathReplace: 1; /* replace slashes, replace dots, empty string */ - unsigned bSPIffNo1stSP: 1; /* replace slashes, replace dots, empty string */ + unsigned bSPIffNo1stSP: 1; /* be a space if 1st pos if field is no space*/ unsigned bCSV: 1; /* format field in CSV (RFC 4180) format */ unsigned bJSON: 1; /* format field JSON escaped */ + unsigned bJSONf: 1; /* format field JSON *field* (n/v pair) */ } options; /* options as bit fields */ } field; } data; -- cgit