added the "jsonf" property replacer option (and fieldname) & bugfix

bugfix: property replacer option "json" could lead to content loss
        message was truncated if escaping was necessary

5 files changed, 181 insertions, 59 deletions

diff --git a/ChangeLog b/ChangeLog
index 732bb8bb..1532c9db 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,8 @@
 ---------------------------------------------------------------------------
 Version 6.3.9  [DEVEL] 2012-04-??
+- added the "jsonf" property replacer option (and fieldname)
+- bugfix: property replacer option "json" could lead to content loss
+  message was truncated if escaping was necessary
 - bugfix: assigned ruleset was lost when using disk queues
   This looked quite hard to diagnose for disk-assisted queues, as the
   pure memory part worked well, but ruleset info was lost for messages
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index 1178b4ca..5ee1d096 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -13,7 +13,7 @@ the value, e.g. by converting all characters to lower case.</p>
 <p>Syslog message properties are used inside templates. They are
 accessed by putting them between percent signs. Properties can be
 modified by the property replacer. The full syntax is as follows:</p>
-<blockquote><b><code>%propname:fromChar:toChar:options%</code></b></blockquote>
+<blockquote><b><code>%propname:fromChar:toChar:options:fieldname%</code></b></blockquote>
 <h2>Available Properties</h2>
 <p><b><code>propname</code></b> is the
 name of the property to access. It is case-insensitive (prior to 3.17.0, they were case-senstive).
@@ -349,7 +349,24 @@ case-insensitive. Currently, the following options are defined:
 <td><b>json</b></td>
 <td>encode the value so that it can be used inside a JSON field. This means
 that several characters (according to the JSON spec) are being escaped, for 
-example US-ASCII LF is replaced by "\n".</td>
+example US-ASCII LF is replaced by "\n".
+The json option cannot be used together with either jsonf or csv options.
+</td>
+</tr>
+<tr>
+<td><b>jsonf</b></td>
+<td><i>(available in 6.3.9+)</i>
+This signifies that the property should be expressed as a json <b>f</b>ield.
+That means not only the property is written, but rather a complete json field in
+the format<br>
+"fieldname"="value"</b>
+where "filedname" is the assigend field name (or the property name if none was assigned)
+and value is the end result of property replacer operation. Note that value supports
+all property replacer options, like substrings, case converson and the like.
+Values are properly json-escaped. However, field names are (currently) not. It is
+expected that proper field names are configured.
+The jsonf option cannot be used together with either json or csv options.
+</td>
 </tr>
 <tr>
 <td valign="top"><b>csv</b></td>
@@ -360,6 +377,7 @@ text, you need to define a proper template. An example is this one:
 <br>$template csvline,"%syslogtag:::csv%,%msg:::csv%"
 <br>Most importantly, you need to provide the commas between the fields
 inside the template.
+The csv option cannot be used together with either json or jsonf options.
 <br><i>This feature was introduced in rsyslog 4.1.6.</i>
 </td>
 </tr>
@@ -465,13 +483,19 @@ Useful for secure pathname generation (with dynafiles).
 them. For example "escape-cc,sp-if-no-1st-sp". If you use conflicting options together,
 the last one will override the previous one. For example, using "escape-cc,drop-cc" will
 use drop-cc and "drop-cc,escape-cc" will use escape-cc mode.
+<h2>Fieldname</h2>
+<p><i>(available in 6.3.9+)</i>
+<p>This field permits to specify a field name for structured-data emitting property replacer
+options. It was initially introduced to support the "jsonf" option, for which it provides
+the capability to set an alternative field name. If it is not specified, it defaults to 
+the property name.
 <h2>Further Links</h2>
 <ul>
 <li>Article on "<a href="rsyslog_recording_pri.html">Recording
 the Priority of Syslog Messages</a>" (describes use of templates
 to record severity and facility of a message)</li>
 <li><a href="rsyslog_conf.html">Configuration file
-syntax</a>, this is where you actually use the property replacer.</li>
+format</a>, this is where you actually use the property replacer.</li>
 </ul>
 <p>[<a href="manual.html">manual index</a>]
 [<a href="rsyslog_conf.html">rsyslog.conf</a>]
diff --git a/runtime/msg.c b/runtime/msg.c
index 1cc5f6b4..9c7a2203 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -263,6 +263,9 @@ static struct {
 	{ UCHAR_CONSTANT("190"), 5},
 	{ UCHAR_CONSTANT("191"), 5}
 	};
+static char hexdigit[16] =
+	{'0', '1', '2', '3', '4', '5', '6', '7', '8',
+	 '9', 'A', 'B', 'C', 'D', 'E', 'F' };
 
 /*syslog facility names (as of RFC5424) */
 static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", "syslog", "lpr",
@@ -423,7 +426,6 @@ resolveDNS(msg_t *pMsg) {
 	uchar fromHostFQDN[NI_MAXHOST];
 	DEFiRet;
 
-dbgprintf("XXXX: in msg/resolveDNS (dnscache)\n");
 	MsgLock(pMsg);
 	CHKiRet(objUse(net, CORE_COMPONENT));
 	if(pMsg->msgFlags & NEEDS_DNSRESOL) {
@@ -2371,81 +2373,67 @@ finalize_it:
 }
 
 
-/* encode a property in JSON escaped format. This is a helper
- * to MsgGetProp. It needs to update all provided parameters.
- * Note: Code is borrowed from libee (my own code, so ASL 2.0
- * is fine with it); this function may later be replaced by
- * some "better" and more complete implementation (maybe from
- * libee or its helpers).
- * For performance reasons, we begin to copy the string only
- * when we recognice that we actually need to do some escaping.
- * rgerhards, 2012-03-16
+/* Encode a JSON value and add it to provided string. Note that 
+ * the string object may be NULL. In this case, it is created
+ * if and only if escaping is needed.
  */
 static rsRetVal
-jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen)
+jsonAddVal(uchar *pSrc, unsigned buflen, es_str_t **dst)
 {
-	static char hexdigit[16] =
-		{'0', '1', '2', '3', '4', '5', '6', '7', '8',
-		 '9', 'A', 'B', 'C', 'D', 'E', 'F' };
 	unsigned char c;
 	es_size_t i;
 	char numbuf[4];
 	int j;
-	unsigned buflen;
-	uchar *pSrc;
-	es_str_t *dst = NULL;
 	DEFiRet;
 
-	pSrc = *ppRes;
-	buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen;
 	for(i = 0 ; i < buflen ; ++i) {
 		c = pSrc[i];
 		if(   (c >= 0x23 && c <= 0x5b)
 		   || (c >= 0x5d /* && c <= 0x10FFFF*/)
 		   || c == 0x20 || c == 0x21) {
 			/* no need to escape */
-			if(dst != NULL)
-				es_addChar(&dst, c);
+			if(*dst != NULL)
+				es_addChar(dst, c);
 		} else {
-			if(dst == NULL) {
+			if(*dst == NULL) {
 				if(i == 0) {
 					/* we hope we have only few escapes... */
-					dst = es_newStr(buflen+10);
+					*dst = es_newStr(buflen+10);
 				} else {
-					dst = es_newStrFromBuf((char*)pSrc, i-1);
+					*dst = es_newStrFromBuf((char*)pSrc, i-1);
 				}
-				if(dst == NULL) {
+				if(*dst == NULL) {
 					ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
 				}
 			}
 			/* we must escape, try RFC4627-defined special sequences first */
 			switch(c) {
 			case '\0':
-				es_addBuf(&dst, "\\u0000", 6);
+				es_addBuf(dst, "\\u0000", 6);
 				break;
 			case '\"':
-				es_addBuf(&dst, "\\\"", 2);
+				es_addBuf(dst, "\\\"", 2);
 				break;
 			case '/':
-				es_addBuf(&dst, "\\/", 2);
+				es_addBuf(dst, "\\/", 2);
 				break;
 			case '\\':
-				es_addBuf(&dst, "\\\\", 2);
+				es_addBuf(dst, "\\\\", 2);
 				break;
 			case '\010':
-				es_addBuf(&dst, "\\b", 2);
+				es_addBuf(dst, "\\b", 2);
 				break;
 			case '\014':
-				es_addBuf(&dst, "\\f", 2);
+				es_addBuf(dst, "\\f", 2);
 				break;
 			case '\n':
-				es_addBuf(&dst, "\\n", 2);
+				es_addBuf(dst, "\\n", 2);
 				break;
 			case '\r':
-				es_addBuf(&dst, "\\r", 2);
+				es_addBuf(dst, "\\r", 2);
 				break;
 			case '\t':
-				es_addBuf(&dst, "\\t", 2);
+				es_addBuf(dst, "\\t", 2);
 				break;
 			default:
 				/* TODO : proper Unicode encoding (see header comment) */
@@ -2453,12 +2441,38 @@ jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen)
 					numbuf[3-j] = hexdigit[c % 16];
 					c = c / 16;
 				}
-				es_addBuf(&dst, "\\u", 2);
-				es_addBuf(&dst, numbuf, 4);
+				es_addBuf(dst, "\\u", 2);
+				es_addBuf(dst, numbuf, 4);
 				break;
 			}
 		}
 	}
+finalize_it:
+	RETiRet;
+}
+
+
+/* encode a property in JSON escaped format. This is a helper
+ * to MsgGetProp. It needs to update all provided parameters.
+ * Note: Code is borrowed from libee (my own code, so ASL 2.0
+ * is fine with it); this function may later be replaced by
+ * some "better" and more complete implementation (maybe from
+ * libee or its helpers).
+ * For performance reasons, we begin to copy the string only
+ * when we recognice that we actually need to do some escaping.
+ * rgerhards, 2012-03-16
+ */
+static rsRetVal
+jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen)
+{
+	unsigned buflen;
+	uchar *pSrc;
+	es_str_t *dst = NULL;
+	DEFiRet;
+
+	pSrc = *ppRes;
+	buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen;
+	CHKiRet(jsonAddVal(pSrc, buflen, &dst));
 
 	if(dst != NULL) {
 		/* we updated the string and need to replace the
@@ -2468,6 +2482,7 @@ jsonEncode(uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen)
 			free(*ppRes);
 		*ppRes = (uchar*)es_str2cstr(dst, NULL);
 		*pbMustBeFreed = 1;
+		*pBufLen = -1;
 		es_deleteStr(dst);
 	}
 
@@ -2476,6 +2491,46 @@ finalize_it:
 }
 
 
+/* Format a property as JSON field, that means
+ * "name"="value"
+ * where value is JSON-escaped (here we assume that the name
+ * only contains characters from the valid character set).
+ * Note: this function duplicates code from jsonEncode(). 
+ * TODO: these two functions should be combined, at least if
+ * that makes any sense from a performance PoV - definitely
+ * something to consider at a later stage. rgerhards, 2012-04-19
+ */
+static rsRetVal
+jsonField(struct templateEntry *pTpe, uchar **ppRes, unsigned short *pbMustBeFreed, int *pBufLen)
+{
+	unsigned buflen;
+	uchar *pSrc;
+	es_str_t *dst = NULL;
+	DEFiRet;
+
+	pSrc = *ppRes;
+	buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen;
+	/* we hope we have only few escapes... */
+	dst = es_newStr(buflen+es_strlen(pTpe->data.field.fieldName)+15);
+	es_addChar(&dst, '"');
+	es_addStr(&dst, pTpe->data.field.fieldName);
+	es_addBufConstcstr(&dst, "\"=\"");
+	CHKiRet(jsonAddVal(pSrc, buflen, &dst));
+	es_addChar(&dst, '"');
+
+	if(*pbMustBeFreed)
+		free(*ppRes);
+	/* we know we do not have \0 chars - so the size does not change */
+	*pBufLen = es_strlen(dst);
+	*ppRes = (uchar*)es_str2cstr(dst, NULL);
+	*pbMustBeFreed = 1;
+	es_deleteStr(dst);
+
+finalize_it:
+	RETiRet;
+}
+
+
 /* This function returns a string-representation of the 
  * requested message property. This is a generic function used
  * to abstract properties so that these can be easier
@@ -3301,6 +3356,8 @@ dbgprintf("prop repl 4, pRes='%s', len %d\n", pRes, bufLen);
 		*pbMustBeFreed = 1;
 	} else if(pTpe->data.field.options.bJSON) {
 		jsonEncode(&pRes, pbMustBeFreed, &bufLen);
+	} else if(pTpe->data.field.options.bJSONf) {
+		jsonField(pTpe, &pRes, pbMustBeFreed, &bufLen);
 	}
 
 	if(bufLen == -1)
@@ -3369,7 +3426,6 @@ msgGetMsgVarNew(msg_t *pThis, uchar *name)
 	propNameStrToID(name, &propid);
 	pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed);
 
-dbgprintf("ZZZZ: var %s returns '%s'\n", name, pszProp);
 	estr = es_newStrFromCStr((char*)pszProp, propLen);
 	if(bMustBeFreed)
 		free(pszProp);
diff --git a/template.c b/template.c
index 2326819c..c6878459 100644
--- a/template.c
+++ b/template.c
@@ -493,18 +493,18 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe)
 
 	p = *pp;
 
-	while(*p && *p != '%') {
+	while(*p && *p != '%' && *p != ':') {
 		/* outer loop - until end of options */
 		i = 0;
 		while((i < sizeof(Buf) / sizeof(char)) &&
-		      *p && *p != '%' && *p != ',') {
+		      *p && *p != '%' && *p != ':' && *p != ',') {
 			/* inner loop - until end of ONE option */
 			Buf[i++] = tolower((int)*p);
 			++p;
 		}
 		Buf[i] = '\0'; /* terminate */
 		/* check if we need to skip oversize option */
-		while(*p && *p != '%' && *p != ',')
+		while(*p && *p != '%' && *p != ':' && *p != ',')
 			++p;	/* just skip */
 		if(*p == ',')
 			++p; /* eat ',' */
@@ -544,19 +544,26 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe)
 		 } else if(!strcmp((char*)Buf, "secpath-replace")) {
 			pTpe->data.field.options.bSecPathReplace = 1;
 		 } else if(!strcmp((char*)Buf, "csv")) {
-		 	if(pTpe->data.field.options.bJSON) {
-				errmsg.LogError(0, NO_ERRCODE, "error: can not specify "
-					"both csv and json options - csv ignored");
+		 	if(pTpe->data.field.options.bJSON || pTpe->data.field.options.bJSONf) {
+				errmsg.LogError(0, NO_ERRCODE, "error: can only specify "
+					"one option out of (json, jsonf, csv) - csv ignored");
 			} else {
 				pTpe->data.field.options.bCSV = 1;
 			}
 		 } else if(!strcmp((char*)Buf, "json")) {
-		 	if(pTpe->data.field.options.bCSV) {
-				errmsg.LogError(0, NO_ERRCODE, "error: can not specify "
-					"both csv and json options - json ignored");
+		 	if(pTpe->data.field.options.bCSV || pTpe->data.field.options.bJSON) {
+				errmsg.LogError(0, NO_ERRCODE, "error: can only specify "
+					"one option out of (json, jsonf, csv) - json ignored");
 			} else {
 				pTpe->data.field.options.bJSON = 1;
 			}
+		 } else if(!strcmp((char*)Buf, "jsonf")) {
+		 	if(pTpe->data.field.options.bCSV || pTpe->data.field.options.bJSON) {
+				errmsg.LogError(0, NO_ERRCODE, "error: can only specify "
+					"one option out of (json, jsonf, csv) - jsonf ignored");
+			} else {
+				pTpe->data.field.options.bJSONf = 1;
+			}
 		 } else {
 			dbgprintf("Invalid field option '%s' specified - ignored.\n", Buf);
 		 }
@@ -573,7 +580,8 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe)
 static int do_Parameter(unsigned char **pp, struct template *pTpl)
 {
 	unsigned char *p;
-	cstr_t *pStrB;
+	cstr_t *pStrProp;
+	cstr_t *pStrField = NULL;
 	struct templateEntry *pTpe;
 	int iNum;	/* to compute numbers */
 #ifdef FEATURE_REGEXP
@@ -590,7 +598,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 
 	p = (unsigned char*) *pp;
 
-	if(cstrConstruct(&pStrB) != RS_RET_OK)
+	if(cstrConstruct(&pStrProp) != RS_RET_OK)
 		 return 1;
 
 	if((pTpe = tpeConstruct(pTpl)) == NULL) {
@@ -601,25 +609,24 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 	pTpe->eEntryType = FIELD;
 
 	while(*p && *p != '%' && *p != ':') {
-		cstrAppendChar(pStrB, tolower(*p));
+		cstrAppendChar(pStrProp, tolower(*p));
 		++p; /* do NOT do this in tolower()! */
 	}
 
 	/* got the name */
-	cstrFinalize(pStrB);
+	cstrFinalize(pStrProp);
 
-	if(propNameToID(pStrB, &pTpe->data.field.propid) != RS_RET_OK) {
-		cstrDestruct(&pStrB);
+	if(propNameToID(pStrProp, &pTpe->data.field.propid) != RS_RET_OK) {
+		cstrDestruct(&pStrProp);
 		return 1;
 	}
 	if(pTpe->data.field.propid == PROP_CEE) {
 		/* in CEE case, we need to preserve the actual property name */
-		if((pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrB)+2, cstrLen(pStrB)-2)) == NULL) {
-			cstrDestruct(&pStrB);
+		if((pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp)+2, cstrLen(pStrProp)-2)) == NULL) {
+			cstrDestruct(&pStrProp);
 			return 1;
 		}
 	}
-	cstrDestruct(&pStrB);
 
 	/* Check frompos, if it has an R, then topos should be a regex */
 	if(*p == ':') {
@@ -873,6 +880,34 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 		doOptions(&p, pTpe);
 	}
 
+	/* check field name */
+	if(*p == ':') {
+		++p; /* eat ':' */
+		if(cstrConstruct(&pStrField) != RS_RET_OK)
+			 return 1;
+		while(*p != ':' && *p != '%' && *p != '\0') {
+			cstrAppendChar(pStrField, *p);
+			++p;
+		}
+		cstrFinalize(pStrField);
+	}
+
+	/* save field name - if none was given, use the property name instead */
+	if(pStrField == NULL) {
+		if((pTpe->data.field.fieldName =
+		      es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp), cstrLen(pStrProp))) == NULL) {
+			return 1;
+		}
+	} else {
+		if((pTpe->data.field.fieldName =
+		      es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrField), cstrLen(pStrField))) == NULL) {
+			return 1;
+		}
+		cstrDestruct(&pStrField);
+	}
+
+	cstrDestruct(&pStrProp);
+
 	if(*p) ++p; /* eat '%' */
 
 	*pp = p;
@@ -1130,6 +1165,8 @@ void tplDeleteAll(rsconf_t *conf)
 				}
 				if(pTpeDel->data.field.propName != NULL)
 					es_deleteStr(pTpeDel->data.field.propName);
+				if(pTpeDel->data.field.fieldName != NULL)
+					es_deleteStr(pTpeDel->data.field.fieldName);
 #endif
 				break;
 			}
diff --git a/template.h b/template.h
index a9eff6b1..f55f64b3 100644
--- a/template.h
+++ b/template.h
@@ -91,6 +91,7 @@ struct templateEntry {
 			int field_expand;	/* use multiple instances of the field delimiter as a single one? */
 
 			es_str_t *propName;	/**< property name (currently being used for CEE only) */
+			es_str_t *fieldName;	/**< field name to be used for structured output */
 
 			enum tplFormatTypes eDateFormat;
 			enum tplFormatCaseConvTypes eCaseConv;
@@ -101,9 +102,10 @@ struct templateEntry {
 				unsigned bDropLastLF: 1;	/* drop last LF char in msg (PIX!) */
 				unsigned bSecPathDrop: 1;	/* drop slashes, replace dots, empty string */
 				unsigned bSecPathReplace: 1;	/* replace slashes, replace dots, empty string */
-				unsigned bSPIffNo1stSP: 1;	/* replace slashes, replace dots, empty string */
+				unsigned bSPIffNo1stSP: 1;	/* be a space if 1st pos if field is no space*/
 				unsigned bCSV: 1;		/* format field in CSV (RFC 4180) format */
 				unsigned bJSON: 1;		/* format field JSON escaped */
+				unsigned bJSONf: 1;		/* format field JSON *field* (n/v pair) */
 			} options;		/* options as bit fields */
 		} field;
 	} data;