Handle $!all-json in field templates

Let msg.c handle appending fields, which also allows it to append more
than one field per replaced property.

The result allows supplementing CEE data with other (non-constant)
properties, but will currently result in duplicates if $!all-json and
the template contain fields with the same names.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>

3 files changed, 97 insertions, 28 deletions

diff --git a/runtime/msg.c b/runtime/msg.c
index 6725203c..0d01f5e1 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -3445,6 +3445,79 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 	return(pRes);
 }
 
+rsRetVal FBSensureSpace(struct fieldBuildingState *state)
+{
+	DEFiRet;
+
+	if(state->nextField == state->allocatedFields) {
+		struct templateField *p;
+		size_t newSize;
+
+		newSize = state->allocatedFields * 2 + 4;
+		CHKmalloc(p = realloc(state->fields,
+				      newSize * sizeof(*state->fields)));
+		state->fields = p;
+		state->allocatedFields = newSize;
+	}
+
+finalize_it:
+	RETiRet;
+}
+
+/* Append fields designated by PROPID to (pFields, pAllocatedFields,
+   pNextField) */
+rsRetVal MsgAppendFields(msg_t *pMsg, struct templateEntry *pTpe,
+			 struct fieldBuildingState *state)
+{
+	DEFiRet;
+	struct templateField *dst;
+	uchar *pVal;
+	size_t propLen;
+	unsigned short bMustBeFreed;
+	struct ee_fieldbucket_listnode *node;
+	struct ee_field *src;
+
+	if(pTpe->data.field.propid == PROP_CEE_ALL_JSON) { /* Special case - multiple fields */
+		if(pMsg->event != NULL && pMsg->event->fields != NULL) {
+			/* FIXME: should there be an libee API allowing enumeration
+			   instead of these direct accesses? */
+			for (node = pMsg->event->fields->root; node != NULL;
+			     node = node->next) {
+				CHKiRet(FBSensureSpace(state));
+
+				src = node->field;
+				dst = state->fields + state->nextField;
+				dst->fieldName = src->name;
+				if (src->nVals == 0) {
+					CHKmalloc(dst->value = (uchar*)strdup(""));
+				} else {
+					/* FIXME: are there other types to consider? */
+					assert(src->val->valtype == ee_valtype_str);
+					/* FIXME: This corrupts embedded NULs - can we do better?  The only current user to consider is mongodb */
+					CHKmalloc(dst->value = (uchar *)es_str2cstr(src->val->val.str, NULL));
+				}
+				state->nextField++;
+			}
+		}
+	} else {
+		CHKiRet(FBSensureSpace(state));
+
+		dst = state->fields + state->nextField;
+		dst->fieldName = pTpe->data.field.fieldName;
+		pVal = MsgGetProp(pMsg, pTpe, pTpe->data.field.propid,
+				  pTpe->data.field.propName, &propLen,
+				  &bMustBeFreed);
+		if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */
+			dst->value = pVal; /* ... so we can use it! */
+		} else {
+			CHKmalloc(dst->value = (uchar*)strdup((char*)pVal));
+		}
+		state->nextField++;
+	}
+finalize_it:
+	RETiRet;
+}
+
 
 /* The function returns a cee variable suitable for use with RainerScript. 
  * Note: caller must free the returned string.
diff --git a/runtime/msg.h b/runtime/msg.h
index f6b54a77..345f2a9e 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -140,6 +140,11 @@ struct msg {
 #define NEEDS_ACLCHK_U	0x080	/* check UDP ACLs after DNS resolution has been done in main queue consumer */
 #define NO_PRI_IN_RAW	0x100	/* rawmsg does not include a PRI (Solaris!), but PRI is already set correctly in the msg object */
 
+struct fieldBuildingState {
+	struct templateField *fields;
+	size_t allocatedFields;
+	size_t nextField;
+};
 
 /* function prototypes
  */
@@ -173,6 +178,9 @@ rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG);
 uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
                   propid_t propid, es_str_t *propName,
 		  size_t *pPropLen, unsigned short *pbMustBeFreed);
+rsRetVal FBSensureSpace(struct fieldBuildingState *state);
+rsRetVal MsgAppendFields(msg_t *pMsg, struct templateEntry *pTpe,
+			 struct fieldBuildingState *state);
 char *textpri(char *pRes, size_t pResLen, int pri);
 rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar);
 es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name);
diff --git a/template.c b/template.c
index d422f8ee..d9ca0710 100644
--- a/template.c
+++ b/template.c
@@ -280,50 +280,38 @@ rsRetVal tplToFields(struct template *pTpl, msg_t *pMsg,
 {
 	DEFiRet;
 	struct templateEntry *pTpe;
-	struct templateField *fields;
-	size_t i, propLen;
-	unsigned short bMustBeFreed;
-	uchar *pVal;
+	struct fieldBuildingState state;
+	size_t i;
 
 	assert(pTpl != NULL);
 	assert(pMsg != NULL);
 	assert(pFields != NULL);
 
-	/* loop through the template. We obtain one value, create a
-	 * private copy (if necessary), add it to the string array
-	 * and then on to the next until we have processed everything.
-	 */
-
-	/* The zeroization implicitly terminates the output array. */
-	CHKmalloc(fields = calloc(pTpl->tpenElements + 1, sizeof(*fields)));
+	state.nextField = 0;
+	state.allocatedFields = pTpl->tpenElements + 1;
+	CHKmalloc(state.fields = malloc(state.allocatedFields
+					* sizeof(*state.fields)));
 
-	i = 0;
 	for(pTpe = pTpl->pEntryRoot; pTpe != NULL; pTpe = pTpe->pNext) {
 		if(pTpe->eEntryType == CONSTANT) {
 			/* Completely ingore this - we don't know a field name
 			   to use anyway. */
 		} else if(pTpe->eEntryType == FIELD) {
-			fields[i].fieldName = pTpe->data.field.fieldName;
-			pVal = MsgGetProp(pMsg, pTpe, pTpe->data.field.propid,
-					  pTpe->data.field.propName, &propLen,
-					  &bMustBeFreed);
-			if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */
-				fields[i].value = pVal; /* ... so we can use it! */
-			} else {
-				CHKmalloc(fields[i].value = (uchar*)strdup((char*) pVal));
-			}
-			i++;
+			CHKiRet(MsgAppendFields(pMsg, pTpe, &state));
 		}
 	}
+	CHKiRet(FBSensureSpace(&state));
+	/* Terminate the array */
+	state.fields[state.nextField].fieldName = NULL;
 
 finalize_it:
-	if(iRet != RS_RET_OK && fields != NULL) {
-		for(i = 0; fields[i].fieldName != NULL; i++)
-			free(fields[i].value);
-		free(fields);
-		fields = NULL;
+	if(iRet != RS_RET_OK && state.fields != NULL) {
+		for(i = 0; i < state.nextField; i++)
+			free(state.fields[i].value);
+		free(state.fields);
+		state.fields = NULL;
 	}
-	*pFields = fields;
+	*pFields = state.fields;
 
 	RETiRet;
 }