From ab7b1b381a467d6a61760a5cb84ef804a74598f1 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Fri, 24 Aug 2012 15:08:12 +0200
Subject: add base plumbing for template() config object to grammar

---
 grammar/grammar.y      | 14 +++++++++++++-
 grammar/lexer.l        |  8 +++++++-
 grammar/rainerscript.h |  3 +++
 runtime/rsconf.c       | 10 ++++++++++
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index 402b1a57..35afae30 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -63,6 +63,9 @@ extern int yyerror(char*);
 %token ENDOBJ
 %token <s> CFSYSLINE
 %token BEGIN_ACTION
+%token BEGIN_PROPERTY
+%token BEGIN_CONSTANT
+%token BEGIN_TPL
 %token STOP
 %token <s> LEGACY_ACTION
 %token <s> PRIFILT
@@ -89,7 +92,7 @@ extern int yyerror(char*);
 %token CMP_STARTSWITHI
 
 %type <nvlst> nv nvlst
-%type <obj> obj
+%type <obj> obj propconst
 %type <actlst> actlst
 %type <actlst> act
 %type <s> cfsysline
@@ -128,6 +131,15 @@ conf:	/* empty (to end recursion) */
 	| conf BSD_HOST_SELECTOR	{ cnfDoBSDHost($2); }
 obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
 	| BEGIN_ACTION nvlst ENDOBJ 	{ $$ = cnfobjNew(CNFOBJ_ACTION, $2); }
+        | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2);  dbgprintf("processing template() without {}\n"); }
+        | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
+					{ $$ = cnfobjNew(CNFOBJ_TPL, $2); dbgprintf("processing template() WITH {}\n"); }
+/* TODO: NOTE:
+   propconst is the NEXT step. It is just included as an experiment and needs
+   to be replaced.
+*/
+propconst: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
+					  dbgprintf("processed property()\n"); }
 cfsysline: CFSYSLINE	 		{ $$ = $1; }
 nvlst:					{ $$ = NULL; }
 	| nvlst nv 			{ $2->next = $1; $$ = $2; }
diff --git a/grammar/lexer.l b/grammar/lexer.l
index e688ffce..c5e7bf7d 100644
--- a/grammar/lexer.l
+++ b/grammar/lexer.l
@@ -9,7 +9,7 @@
   * cases. So while we hope that cfsysline support can be dropped some time in
   * the future, we will probably keep these useful constructs.
   *
-  * Copyright 2011 Rainer Gerhards and Adiscon GmbH.
+  * Copyright 2011-2012 Rainer Gerhards and Adiscon GmbH.
   *
   * This file is part of the rsyslog runtime library.
   *
@@ -151,6 +151,12 @@ int fileno(FILE *stream);
 				  BEGIN INITIAL; }
 "global"[ \n\t]*"("		{ yylval.objType = CNFOBJ_GLOBAL;
 				  BEGIN INOBJ; return BEGINOBJ; }
+"template"[ \n\t]*"("		{ yylval.objType = CNFOBJ_TPL;
+				  BEGIN INOBJ; return BEGIN_TPL; }
+"property"[ \n\t]*"("		{ yylval.objType = CNFOBJ_PROPERTY;
+				  BEGIN INOBJ; return BEGIN_PROPERTY; }
+"constant"[ \n\t]*"("		{ yylval.objType = CNFOBJ_CONSTANT;
+				  BEGIN INOBJ; return BEGIN_CONSTANT; }
 "input"[ \n\t]*"("		{ yylval.objType = CNFOBJ_INPUT;
 				  BEGIN INOBJ; return BEGINOBJ; }
 "module"[ \n\t]*"("		{ yylval.objType = CNFOBJ_MODULE;
diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h
index 5ff71bee..83a253f7 100644
--- a/grammar/rainerscript.h
+++ b/grammar/rainerscript.h
@@ -16,6 +16,9 @@ enum cnfobjType {
 	CNFOBJ_GLOBAL,
 	CNFOBJ_INPUT,
 	CNFOBJ_MODULE,
+	CNFOBJ_TPL,
+	CNFOBJ_PROPERTY,
+	CNFOBJ_CONSTANT,
 	CNFOBJ_INVALID = 0
 };
 
diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index bd002353..f07ab314 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -386,6 +386,7 @@ yyerror(char *s)
 }
 void cnfDoObj(struct cnfobj *o)
 {
+int bChkUnuse = 1; // TODO: Delete
 	dbgprintf("cnf:global:obj: ");
 	cnfobjPrint(o);
 	switch(o->objType) {
@@ -398,7 +399,16 @@ void cnfDoObj(struct cnfobj *o)
 	case CNFOBJ_ACTION:
 		actionProcessCnf(o);
 		break;
+	case CNFOBJ_TPL:
+		//processTemplate(o);
+bChkUnuse = 0;
+		break;
+	case CNFOBJ_PROPERTY:
+		//processTemplate(o);
+bChkUnuse = 0;
+		break;
 	}
+if(bChkUnuse)
 	nvlstChkUnused(o->nvlst);
 	cnfobjDestruct(o);
 }
-- 
cgit 


From 1eac94e11dab1e7caead5e31a57d2cae31b5ad62 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 11:08:38 +0200
Subject: v6 config/templates: legacy types are now supported via template()

---
 runtime/rsconf.c |   4 +-
 template.c       | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 template.h       |   1 +
 3 files changed, 180 insertions(+), 3 deletions(-)

diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index f07ab314..cbc09c11 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -66,6 +66,7 @@
 #include "parserif.h"
 #include "modules.h"
 #include "dirty.h"
+#include "template.h"
 
 /* static data */
 DEFobjStaticHelpers
@@ -400,8 +401,7 @@ int bChkUnuse = 1; // TODO: Delete
 		actionProcessCnf(o);
 		break;
 	case CNFOBJ_TPL:
-		//processTemplate(o);
-bChkUnuse = 0;
+		tplProcessCnf(o);
 		break;
 	case CNFOBJ_PROPERTY:
 		//processTemplate(o);
diff --git a/template.c b/template.c
index 32b8d561..01eab556 100644
--- a/template.c
+++ b/template.c
@@ -2,7 +2,7 @@
  * Please see syslogd.c for license information.
  * begun 2004-11-17 rgerhards
  *
- * Copyright 2004, 2007 Rainer Gerhards and Adiscon
+ * Copyright 2004-2012 Rainer Gerhards and Adiscon
  *
  * This file is part of rsyslog.
  *
@@ -45,6 +45,22 @@ DEFobjCurrIf(obj)
 DEFobjCurrIf(errmsg)
 DEFobjCurrIf(strgen)
 
+/* tables for interfacing with the v6 config system */
+static struct cnfparamdescr cnfparamdescr[] = {
+	{ "name", eCmdHdlrString, 1 },
+	{ "type", eCmdHdlrString, 0 },
+	{ "string", eCmdHdlrString, 0 },
+	{ "plugin", eCmdHdlrString, 0 },
+	{ "option.stdsql", eCmdHdlrBinary, 0 },
+	{ "option.sql", eCmdHdlrBinary, 0 },
+	{ "option.json", eCmdHdlrBinary, 0 }
+};
+static struct cnfparamblk pblk =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(cnfparamdescr)/sizeof(struct cnfparamdescr),
+	  cnfparamdescr
+	};
+
 #ifdef FEATURE_REGEXP
 DEFobjCurrIf(regexp)
 static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */
@@ -1122,6 +1138,166 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin
 }
 
 
+
+// v6 - ASL 2.0!
+/* Add a new template via the v6 config system.
+ */
+rsRetVal
+tplProcessCnf(struct cnfobj *o)
+{
+	struct template *pTpl = NULL;
+	struct cnfparamvals *pvals;
+	int lenName;
+	char *name = NULL;
+	uchar *tplStr = NULL;
+	uchar *p;
+	uchar *plugin;
+	enum { T_STRING, T_PLUGIN, T_LIST } tplType;
+	int i;
+	int o_sql=0, o_stdsql=0, o_json=0; /* options */
+	int numopts;
+	rsRetVal localRet;
+	DEFiRet;
+
+	pvals = nvlstGetParams(o->nvlst, &pblk, NULL);
+	cnfparamsPrint(&pblk, pvals);
+
+	
+	for(i = 0 ; i < pblk.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(pblk.descr[i].name, "name")) {
+			lenName = es_strlen(pvals[i].val.d.estr);
+			name = es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblk.descr[i].name, "type")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"string", sizeof("string")-1)) {
+				tplType = T_STRING;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"plugin", sizeof("plugin")-1)) {
+				tplType = T_PLUGIN;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"list", sizeof("list")-1)) {
+				tplType = T_LIST;
+				errmsg.LogError(0, RS_RET_ERR, "template type 'list' is not "
+					"supported in this rsyslog version");
+				ABORT_FINALIZE(RS_RET_ERR);
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid template type '%s'",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblk.descr[i].name, "string")) {
+			tplStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblk.descr[i].name, "plugin")) {
+			plugin = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblk.descr[i].name, "option.stdsql")) {
+			o_stdsql = pvals[i].val.d.n;
+		} else if(!strcmp(pblk.descr[i].name, "option.sql")) {
+			o_sql = pvals[i].val.d.n;
+		} else if(!strcmp(pblk.descr[i].name, "option.json")) {
+			o_json = pvals[i].val.d.n;
+		} else {
+			dbgprintf("template: program error, non-handled "
+			  "param '%s'\n", pblk.descr[i].name);
+		}
+	}
+
+	/* do config sanity checks */
+	if(tplStr  == NULL) {
+		if(tplType == T_STRING) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type string needs "
+				"string parameter", name);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(tplType != T_STRING) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a string "
+				"template but has a string specified - ignored", name);
+		}
+	}
+
+	if(plugin  == NULL) {
+		if(tplType == T_PLUGIN) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type plugin needs "
+				"plugin parameter - ignored", name);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(tplType != T_PLUGIN) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a plugin "
+				"template but has a plugin specified - ignored", name);
+		}
+	}
+
+	numopts = 0;
+	if(o_sql) ++numopts;
+	if(o_stdsql) ++numopts;
+	if(o_json) ++numopts;
+	if(numopts > 1) {
+		errmsg.LogError(0, RS_RET_ERR, "template '%s' has multiple incompatible "
+			"options of sql, stdsql or json specified", name);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	/* config ok */
+	if((pTpl = tplConstruct(loadConf)) == NULL) {
+		DBGPRINTF("template.c: tplConstruct failed!\n");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	pTpl->pszName = name;
+	pTpl->iLenName = lenName;
+	
+	switch(tplType) {
+	case T_STRING:	p = tplStr;
+			while(*p) {
+				switch(*p) {
+					case '%': /* parameter */
+						++p; /* eat '%' */
+						do_Parameter(&p, pTpl);
+						break;
+					default: /* constant */
+						do_Constant(&p, pTpl);
+						break;
+				}
+			}
+			break;
+	case T_PLUGIN:	p = plugin;
+			/* TODO: the use of tplAddTplMod() can be improved! */
+			localRet = tplAddTplMod(pTpl, &p);
+			if(localRet != RS_RET_OK) {
+				errmsg.LogError(0, localRet, "template '%s': error %d "
+						"defining template via plugin (strgen) module",
+						pTpl->pszName, localRet);
+				ABORT_FINALIZE(localRet);
+			}
+	}
+	
+	pTpl->optFormatEscape = NO_ESCAPE;
+	if(o_stdsql)
+		pTpl->optFormatEscape = STDSQL_ESCAPE;
+	else if(o_sql)
+		pTpl->optFormatEscape = SQL_ESCAPE;
+	else if(o_json)
+		pTpl->optFormatEscape = JSON_ESCAPE;
+
+finalize_it:
+	if(iRet != RS_RET_OK) {
+		if(pTpl != NULL) {
+			/* we simply make the template defunct in this case by setting
+			 * its name to a zero-string. We do not free it, as this would
+			 * require additional code and causes only a very small memory
+			 * consumption. TODO: maybe in next iteration...
+			 */
+			*pTpl->pszName = '\0';
+		}
+	}
+
+	RETiRet;
+}
+
+// END v6
+
+
 /* Find a template object based on name. Search
  * currently is case-senstive (should we change?).
  * returns pointer to template object if found and
diff --git a/template.h b/template.h
index e41823e9..9daf2c6b 100644
--- a/template.h
+++ b/template.h
@@ -141,6 +141,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *)
 rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode);
 
 rsRetVal templateInit();
+rsRetVal tplProcessCnf(struct cnfobj *o);
 
 #endif /* #ifndef TEMPLATE_H_INCLUDED */
 /* vim:set ai:
-- 
cgit 


From 6258cb42fd407b9388de63c746634b4df03e78eb Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 13:30:53 +0200
Subject: milestone: base plumbing for LIST-type templates mostly in place

---
 grammar/grammar.y      | 22 +++++++++++++++-------
 grammar/rainerscript.c | 39 +++++++++++++++++++++++++++++++++++++++
 grammar/rainerscript.h | 18 ++++++++++++++++++
 runtime/rsconf.c       |  1 +
 template.c             | 44 +++++++++++++++++++++++++++++++++-----------
 5 files changed, 106 insertions(+), 18 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index 35afae30..d91eb3bb 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -49,6 +49,7 @@ extern int yyerror(char*);
 	enum cnfobjType objType;
 	struct cnfobj *obj;
 	struct nvlst *nvlst;
+	struct objlst *objlst;
 	struct cnfactlst *actlst;
 	struct cnfexpr *expr;
 	struct cnfrule *rule;
@@ -92,7 +93,8 @@ extern int yyerror(char*);
 %token CMP_STARTSWITHI
 
 %type <nvlst> nv nvlst
-%type <obj> obj propconst
+%type <obj> obj property constant
+%type <objlst> propconst
 %type <actlst> actlst
 %type <actlst> act
 %type <s> cfsysline
@@ -133,13 +135,19 @@ obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
 	| BEGIN_ACTION nvlst ENDOBJ 	{ $$ = cnfobjNew(CNFOBJ_ACTION, $2); }
         | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2);  dbgprintf("processing template() without {}\n"); }
         | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
-					{ $$ = cnfobjNew(CNFOBJ_TPL, $2); dbgprintf("processing template() WITH {}\n"); }
-/* TODO: NOTE:
-   propconst is the NEXT step. It is just included as an experiment and needs
-   to be replaced.
-*/
-propconst: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
+					{ $$ = cnfobjNew(CNFOBJ_TPL, $2);
+					  $$->subobjs = $5;
+					  dbgprintf("processing template() WITH {}\n"); }
+propconst:				{ $$ = NULL; }
+	| propconst property		{ if($1 == NULL)
+						$$ = objlstNew($2);
+					  else
+						$1->next = objlstNew($2); }
+	| propconst constant		{ /*$2->next = $1; $$ = $2;*/ }
+property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
 					  dbgprintf("processed property()\n"); }
+constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2);
+					  dbgprintf("processed constant()\n"); }
 cfsysline: CFSYSLINE	 		{ $$ = $1; }
 nvlst:					{ $$ = NULL; }
 	| nvlst nv 			{ $2->next = $1; $$ = $2; }
diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c
index 3bfb2e07..1ccac575 100644
--- a/grammar/rainerscript.c
+++ b/grammar/rainerscript.c
@@ -98,6 +98,44 @@ readConfFile(FILE *fp, es_str_t **str)
 	es_addChar(str, '\0');
 }
 
+struct objlst*
+objlstNew(struct cnfobj *o)
+{
+	struct objlst *lst;
+
+	if((lst = malloc(sizeof(struct objlst))) != NULL) {
+		lst->next = NULL;
+		lst->obj = o;
+	}
+dbgprintf("AAAA: creating new objlst\n");
+cnfobjPrint(o);
+
+	return lst;
+}
+
+void
+objlstDestruct(struct objlst *lst)
+{
+	struct objlst *toDel;
+
+	while(lst != NULL) {
+		toDel = lst;
+		lst = lst->next;
+		// TODO: delete object
+		free(toDel);
+	}
+}
+
+void
+objlstPrint(struct objlst *lst)
+{
+	dbgprintf("objlst %p:\n", lst);
+	while(lst != NULL) {
+		cnfobjPrint(lst->obj);
+		lst = lst->next;
+	}
+}
+
 struct nvlst*
 nvlstNew(es_str_t *name, es_str_t *value)
 {
@@ -581,6 +619,7 @@ cnfobjNew(enum cnfobjType objType, struct nvlst *lst)
 		nvlstChkDupes(lst);
 		o->objType = objType;
 		o->nvlst = lst;
+		o->subobjs = NULL;
 	}
 
 	return o;
diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h
index 83a253f7..4c625cd8 100644
--- a/grammar/rainerscript.h
+++ b/grammar/rainerscript.h
@@ -38,6 +38,15 @@ cnfobjType2str(enum cnfobjType ot)
 	case CNFOBJ_MODULE:
 		return "module";
 		break;
+	case CNFOBJ_TPL:
+		return "template";
+		break;
+	case CNFOBJ_PROPERTY:
+		return "property";
+		break;
+	case CNFOBJ_CONSTANT:
+		return "constant";
+		break;
 	default:return "error: invalid cnfobjType";
 	}
 }
@@ -63,6 +72,12 @@ struct var {
 struct cnfobj {
 	enum cnfobjType objType;
 	struct nvlst *nvlst;
+	struct objlst *subobjs;
+};
+
+struct objlst {
+	struct objlst *next;
+	struct cnfobj *obj;
 };
 
 struct nvlst {
@@ -221,6 +236,9 @@ struct cnfparamvals { /* the values we obtained for param descr. */
 
 int cnfParseBuffer(char *buf, unsigned lenBuf);
 void readConfFile(FILE *fp, es_str_t **str);
+struct objlst* objlstNew(struct cnfobj *obj);
+void objlstDestruct(struct objlst *lst);
+void objlstPrint(struct objlst *lst);
 struct nvlst* nvlstNew(es_str_t *name, es_str_t *value);
 void nvlstDestruct(struct nvlst *lst);
 void nvlstPrint(struct nvlst *lst);
diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index cbc09c11..9c6bf00e 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -404,6 +404,7 @@ int bChkUnuse = 1; // TODO: Delete
 		tplProcessCnf(o);
 		break;
 	case CNFOBJ_PROPERTY:
+	case CNFOBJ_CONSTANT:
 		//processTemplate(o);
 bChkUnuse = 0;
 		break;
diff --git a/template.c b/template.c
index 50db86a3..39f95967 100644
--- a/template.c
+++ b/template.c
@@ -1146,11 +1146,23 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin
 	return(pTpl);
 }
 
+/* create a template in list mode, is build from sub-objects */
+static rsRetVal
+createListTpl(struct template *pTpl, struct cnfobj *o)
+{
+	struct objlst *lst;
+	DEFiRet;
 
+	dbgprintf("AAAA: create template from subobjs\n");
+	objlstPrint(o->subobjs);
 
-// v6 - ASL 2.0!
-/* Add a new template via the v6 config system.
- */
+	for(lst = o->subobjs ; lst != NULL ; lst = lst->next) {
+		dbgprintf("AAAA: subjobject entry %p\n", lst);
+	}
+	RETiRet;
+}
+
+/* Add a new template via the v6 config system.  */
 rsRetVal
 tplProcessCnf(struct cnfobj *o)
 {
@@ -1159,8 +1171,8 @@ tplProcessCnf(struct cnfobj *o)
 	int lenName;
 	char *name = NULL;
 	uchar *tplStr = NULL;
+	uchar *plugin = NULL;
 	uchar *p;
-	uchar *plugin;
 	enum { T_STRING, T_PLUGIN, T_LIST } tplType;
 	int i;
 	int o_sql=0, o_stdsql=0, o_json=0; /* options */
@@ -1170,7 +1182,6 @@ tplProcessCnf(struct cnfobj *o)
 
 	pvals = nvlstGetParams(o->nvlst, &pblk, NULL);
 	cnfparamsPrint(&pblk, pvals);
-
 	
 	for(i = 0 ; i < pblk.nParams ; ++i) {
 		if(!pvals[i].bUsed)
@@ -1185,9 +1196,6 @@ tplProcessCnf(struct cnfobj *o)
 				tplType = T_PLUGIN;
 			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"list", sizeof("list")-1)) {
 				tplType = T_LIST;
-				errmsg.LogError(0, RS_RET_ERR, "template type 'list' is not "
-					"supported in this rsyslog version");
-				ABORT_FINALIZE(RS_RET_ERR);
 			} else {
 				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
 				errmsg.LogError(0, RS_RET_ERR, "invalid template type '%s'",
@@ -1228,7 +1236,7 @@ tplProcessCnf(struct cnfobj *o)
 	if(plugin  == NULL) {
 		if(tplType == T_PLUGIN) {
 			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type plugin needs "
-				"plugin parameter - ignored", name);
+				"plugin parameter", name);
 			ABORT_FINALIZE(RS_RET_ERR);
 		}
 	} else {
@@ -1238,6 +1246,19 @@ tplProcessCnf(struct cnfobj *o)
 		}
 	}
 
+	if(o->subobjs  == NULL) {
+		if(tplType == T_LIST) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type list has "
+				"has no parameters specified", name);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(tplType != T_LIST) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a list "
+				"template but has parameters specified - ignored", name);
+		}
+	}
+
 	numopts = 0;
 	if(o_sql) ++numopts;
 	if(o_stdsql) ++numopts;
@@ -1279,6 +1300,9 @@ tplProcessCnf(struct cnfobj *o)
 						pTpl->pszName, localRet);
 				ABORT_FINALIZE(localRet);
 			}
+			break;
+	case T_LIST:	createListTpl(pTpl, o);
+			break;
 	}
 	
 	pTpl->optFormatEscape = NO_ESCAPE;
@@ -1304,8 +1328,6 @@ finalize_it:
 	RETiRet;
 }
 
-// END v6
-
 
 /* Find a template object based on name. Search
  * currently is case-senstive (should we change?).
-- 
cgit 


From 76e9968c0890cb4db068c953db6e6574b6c28da0 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 17:17:27 +0200
Subject: milestone: LIST-type templates work, but no all options yet present

---
 grammar/grammar.y      |   9 +-
 grammar/rainerscript.c |  18 ++++
 runtime/rsconf.c       |  11 +-
 template.c             | 266 ++++++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 291 insertions(+), 13 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index d91eb3bb..27d0e1d5 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -137,13 +137,10 @@ obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
         | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
 					{ $$ = cnfobjNew(CNFOBJ_TPL, $2);
 					  $$->subobjs = $5;
-					  dbgprintf("processing template() WITH {}\n"); }
+					  dbgprintf("processing template() WITH {}, subobj=%p\n", $5); }
 propconst:				{ $$ = NULL; }
-	| propconst property		{ if($1 == NULL)
-						$$ = objlstNew($2);
-					  else
-						$1->next = objlstNew($2); }
-	| propconst constant		{ /*$2->next = $1; $$ = $2;*/ }
+	| propconst property		{ $$ = objlstAdd($1, $2); }
+	| propconst constant		{ $$ = objlstAdd($1, $2); }
 property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
 					  dbgprintf("processed property()\n"); }
 constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2);
diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c
index 1ccac575..33630a76 100644
--- a/grammar/rainerscript.c
+++ b/grammar/rainerscript.c
@@ -113,6 +113,24 @@ cnfobjPrint(o);
 	return lst;
 }
 
+/* add object to end of object list, always returns pointer to root object */
+struct objlst*
+objlstAdd(struct objlst *root, struct cnfobj *o)
+{
+	struct objlst *l;
+	struct objlst *newl;
+	
+	newl = objlstNew(o);
+	if(root == 0) {
+		root = newl;
+	} else { /* find last, linear search ok, as only during config phase */
+		for(l = root ; l->next != NULL ; l = l->next)
+			;
+		l->next = newl;
+	}
+	return root;
+}
+
 void
 objlstDestruct(struct objlst *lst)
 {
diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index 9c6bf00e..5d2407ec 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -387,7 +387,8 @@ yyerror(char *s)
 }
 void cnfDoObj(struct cnfobj *o)
 {
-int bChkUnuse = 1; // TODO: Delete
+	int bChkUnuse = 1;
+
 	dbgprintf("cnf:global:obj: ");
 	cnfobjPrint(o);
 	switch(o->objType) {
@@ -405,12 +406,12 @@ int bChkUnuse = 1; // TODO: Delete
 		break;
 	case CNFOBJ_PROPERTY:
 	case CNFOBJ_CONSTANT:
-		//processTemplate(o);
-bChkUnuse = 0;
+		/* these types are processed at a later stage */
+		bChkUnuse = 0;
 		break;
 	}
-if(bChkUnuse)
-	nvlstChkUnused(o->nvlst);
+	if(bChkUnuse)
+		nvlstChkUnused(o->nvlst);
 	cnfobjDestruct(o);
 }
 
diff --git a/template.c b/template.c
index 39f95967..cdcec305 100644
--- a/template.c
+++ b/template.c
@@ -66,6 +66,35 @@ static struct cnfparamblk pblk =
 	  cnfparamdescr
 	};
 
+static struct cnfparamdescr cnfparamdescrProperty[] = {
+	{ "name", eCmdHdlrString, 1 },
+	{ "outname", eCmdHdlrString, 0 },
+	{ "dateformat", eCmdHdlrString, 0 },
+	{ "caseconversion", eCmdHdlrString, 0 },
+	{ "controlcharacters", eCmdHdlrString, 0 },
+	{ "securepath", eCmdHdlrString, 0 },
+	{ "format", eCmdHdlrString, 0 },
+	{ "droplastlf", eCmdHdlrBinary, 0 },
+	{ "spifno1stsp", eCmdHdlrBinary, 0 }
+};
+static struct cnfparamblk pblkProperty =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(cnfparamdescrProperty)/sizeof(struct cnfparamdescr),
+	  cnfparamdescrProperty
+	};
+
+static struct cnfparamdescr cnfparamdescrConstant[] = {
+	{ "value", eCmdHdlrString, 1 },
+	{ "outname", eCmdHdlrString, 0 },
+	{ "format", eCmdHdlrString, 0 }
+};
+static struct cnfparamblk pblkConstant =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(cnfparamdescrConstant)/sizeof(struct cnfparamdescr),
+	  cnfparamdescrConstant
+	};
+
+
 #ifdef FEATURE_REGEXP
 DEFobjCurrIf(regexp)
 static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */
@@ -1146,6 +1175,227 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin
 	return(pTpl);
 }
 
+static rsRetVal
+createConstantTpe(struct template *pTpl, struct cnfobj *o)
+{
+	struct templateEntry *pTpe;
+	es_str_t *value;
+	int i;
+	struct cnfparamvals *pvals;
+	DEFiRet;
+
+	/* pull params */
+	pvals = nvlstGetParams(o->nvlst, &pblkConstant, NULL);
+	cnfparamsPrint(&pblkConstant, pvals);
+	
+	for(i = 0 ; i < pblkConstant.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(pblkConstant.descr[i].name, "value")) {
+			value = pvals[i].val.d.estr;
+		} else if(!strcmp(pblkConstant.descr[i].name, "format")) {
+			errmsg.LogError(0, RS_RET_ERR, "paramter 'format' is currently not "
+				"supported for 'constant' template parts - ignored");
+		} else if(!strcmp(pblkConstant.descr[i].name, "outname")) {
+			errmsg.LogError(0, RS_RET_ERR, "paramter 'outname' is currently not "
+				"supported for 'constant' template parts - ignored");
+		} else {
+			dbgprintf("template:constantTpe: program error, non-handled "
+			  "param '%s'\n", pblkConstant.descr[i].name);
+		}
+	}
+
+	/* sanity check */
+
+	/* apply */
+	CHKmalloc(pTpe = tpeConstruct(pTpl));
+	es_unescapeStr(value);
+	pTpe->eEntryType = CONSTANT;
+	pTpe->data.constant.iLenConstant = es_strlen(value);
+	pTpe->data.constant.pConstant = (uchar*)es_str2cstr(value, NULL);
+
+finalize_it:
+	RETiRet;
+}
+
+static rsRetVal
+createPropertyTpe(struct template *pTpl, struct cnfobj *o)
+{
+	struct templateEntry *pTpe;
+	cstr_t *name;
+	es_str_t *estrname;
+	es_str_t *outname = NULL;
+	int i;
+	int droplastlf = 0;
+	int spifno1stsp = 0;
+	struct cnfparamvals *pvals;
+	enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE;
+	enum {CC_NONE, CC_ESCAPE, CC_SPACE, CC_DROP} controlchr = CC_NONE;
+	enum {SP_NONE, SP_DROP, SP_REPLACE} secpath = SP_NONE;
+	enum tplFormatCaseConvTypes caseconv = tplCaseConvNo;
+	enum tplFormatTypes datefmt = tplFmtDefault;
+	DEFiRet;
+
+	/* pull params */
+	pvals = nvlstGetParams(o->nvlst, &pblkProperty, NULL);
+	cnfparamsPrint(&pblkProperty, pvals);
+	
+	for(i = 0 ; i < pblkProperty.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(pblkProperty.descr[i].name, "name")) {
+			estrname = es_strdup(pvals[i].val.d.estr);
+			/* TODO: unify strings!!! */
+			rsCStrConstructFromszStr(&name,
+				(uchar*)es_str2cstr(pvals[i].val.d.estr, NULL));
+		} else if(!strcmp(pblkProperty.descr[i].name, "droplastlf")) {
+			droplastlf = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "spifno1stsp")) {
+			spifno1stsp = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "outname")) {
+			outname = es_strdup(pvals[i].val.d.estr);
+		} else if(!strcmp(pblkProperty.descr[i].name, "format")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"csv", sizeof("csv")-1)) {
+				formatType = F_CSV;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"json", sizeof("json")-1)) {
+				formatType = F_JSON;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"jsonf", sizeof("jsonf")-1)) {
+				formatType = F_JSONF;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid format type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "controlcharacters")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"escape", sizeof("escape")-1)) {
+				controlchr = CC_ESCAPE;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"space", sizeof("space")-1)) {
+				controlchr = CC_SPACE;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"drop", sizeof("drop")-1)) {
+				controlchr = CC_DROP;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid controlcharacter mode '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "securepath")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"drop", sizeof("drop")-1)) {
+				secpath = SP_DROP;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"replace", sizeof("replace")-1)) {
+				secpath = SP_REPLACE;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid securepath mode '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "caseconversion")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"lower", sizeof("lower")-1)) {
+				caseconv = tplCaseConvLower;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"upper", sizeof("upper")-1)) {
+				caseconv = tplCaseConvUpper;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid caseconversion type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "dateformat")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"mysql", sizeof("mysql")-1)) {
+				datefmt = tplFmtMySQLDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"pgsql", sizeof("pgsql")-1)) {
+				datefmt = tplFmtPgSQLDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"rfc3164", sizeof("rfc3164")-1)) {
+				datefmt = tplFmtRFC3164Date;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"rfc3164-buggyday", sizeof("rfc3164-buggyday")-1)) {
+				datefmt = tplFmtRFC3164BuggyDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"rfc3339", sizeof("rfc3339")-1)) {
+				datefmt = tplFmtRFC3339Date;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"unixtimestamp", sizeof("unixtimestamp")-1)) {
+				datefmt = tplFmtUnixDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"subseconds", sizeof("subseconds")-1)) {
+				datefmt = tplFmtSecFrac;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid date format '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else {
+			dbgprintf("template:propertyTpe: program error, non-handled "
+			  "param '%s'\n", pblkProperty.descr[i].name);
+		}
+	}
+	if(outname == NULL)
+		outname = es_strdup(estrname);
+
+	/* sanity check */
+
+	/* apply */
+	CHKmalloc(pTpe = tpeConstruct(pTpl));
+	pTpe->eEntryType = FIELD;
+	CHKiRet(propNameToID(name, &pTpe->data.field.propid));
+	if(pTpe->data.field.propid == PROP_CEE) {
+		/* in CEE case, we need to preserve the actual property name */
+		pTpe->data.field.propName = estrname;
+	} else {
+		es_deleteStr(estrname);
+	}
+	pTpe->data.field.options.bDropLastLF = droplastlf;
+	pTpe->data.field.options.bSPIffNo1stSP = spifno1stsp;
+	pTpe->data.field.eCaseConv = caseconv;
+	switch(formatType) {
+	case F_NONE:
+		/* all set ;) */
+		break;
+	case F_CSV:
+		pTpe->data.field.options.bCSV = 1;
+		break;
+	case F_JSON:
+		pTpe->data.field.options.bJSON = 1;
+		break;
+	case F_JSONF:
+		pTpe->data.field.options.bJSONf = 1;
+		break;
+	}
+	switch(controlchr) {
+	case CC_NONE:
+		/* all set ;) */
+		break;
+	case CC_ESCAPE:
+		pTpe->data.field.options.bEscapeCC = 1;
+		break;
+	case CC_SPACE:
+		pTpe->data.field.options.bSpaceCC = 1;
+		break;
+	case CC_DROP:
+		pTpe->data.field.options.bDropCC = 1;
+		break;
+	}
+	switch(secpath) {
+	case SP_NONE:
+		/* all set ;) */
+		break;
+	case SP_DROP:
+		pTpe->data.field.options.bSecPathDrop = 1;
+		break;
+	case SP_REPLACE:
+		pTpe->data.field.options.bSecPathReplace = 1;
+		break;
+	}
+	pTpe->data.field.fieldName = outname;
+	pTpe->data.field.eDateFormat = datefmt;
+finalize_it:
+	RETiRet;
+}
+
 /* create a template in list mode, is build from sub-objects */
 static rsRetVal
 createListTpl(struct template *pTpl, struct cnfobj *o)
@@ -1153,12 +1403,24 @@ createListTpl(struct template *pTpl, struct cnfobj *o)
 	struct objlst *lst;
 	DEFiRet;
 
-	dbgprintf("AAAA: create template from subobjs\n");
+	dbgprintf("create template from subobjs\n");
 	objlstPrint(o->subobjs);
 
 	for(lst = o->subobjs ; lst != NULL ; lst = lst->next) {
-		dbgprintf("AAAA: subjobject entry %p\n", lst);
+		switch(lst->obj->objType) {
+		case CNFOBJ_PROPERTY:
+			CHKiRet(createPropertyTpe(pTpl, lst->obj));
+			break;
+		case CNFOBJ_CONSTANT:
+			CHKiRet(createConstantTpe(pTpl, lst->obj));
+			break;
+		default:dbgprintf("program error: invalid object type %d "
+				  "in createLstTpl\n", lst->obj->objType);
+			break;
+		}
+		nvlstChkUnused(lst->obj->nvlst);
 	}
+finalize_it:
 	RETiRet;
 }
 
-- 
cgit 


From a36b7131ab79204f43c400de7b5663613af62cc6 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 17:48:16 +0200
Subject: milestone: LIST-type template now only missing regex support

---
 template.c | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/template.c b/template.c
index cdcec305..7a4c398f 100644
--- a/template.c
+++ b/template.c
@@ -74,6 +74,17 @@ static struct cnfparamdescr cnfparamdescrProperty[] = {
 	{ "controlcharacters", eCmdHdlrString, 0 },
 	{ "securepath", eCmdHdlrString, 0 },
 	{ "format", eCmdHdlrString, 0 },
+// From here
+	{ "position.from", eCmdHdlrInt, 0 },
+	{ "position.to", eCmdHdlrInt, 0 },
+	{ "field.number", eCmdHdlrInt, 0 },
+	{ "field.delimiter", eCmdHdlrInt, 0 },
+	{ "regex.expression", eCmdHdlrString, 0 },
+	{ "regex.type", eCmdHdlrString, 0 },
+	{ "regex.nomatchmode", eCmdHdlrString, 0 },
+	{ "regex.match", eCmdHdlrInt, 0 },
+	{ "regex.submatch", eCmdHdlrInt, 0 },
+//-- to here
 	{ "droplastlf", eCmdHdlrBinary, 0 },
 	{ "spifno1stsp", eCmdHdlrBinary, 0 }
 };
@@ -1228,6 +1239,10 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	int i;
 	int droplastlf = 0;
 	int spifno1stsp = 0;
+	int frompos = -1;
+	int topos = -1;
+	int fieldnum = -1;
+	int fielddelim = 9; /* default is HT (USACSII 9) */
 	struct cnfparamvals *pvals;
 	enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE;
 	enum {CC_NONE, CC_ESCAPE, CC_SPACE, CC_DROP} controlchr = CC_NONE;
@@ -1254,6 +1269,14 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 			spifno1stsp = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "outname")) {
 			outname = es_strdup(pvals[i].val.d.estr);
+		} else if(!strcmp(pblkProperty.descr[i].name, "position.from")) {
+			frompos = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "position.to")) {
+			topos = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "field.number")) {
+			fieldnum = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "field.delimiter")) {
+			fielddelim = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "format")) {
 			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"csv", sizeof("csv")-1)) {
 				formatType = F_CSV;
@@ -1337,6 +1360,15 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		outname = es_strdup(estrname);
 
 	/* sanity check */
+	if(topos == -1 && frompos != -1)
+		topos = 2000000000; /* large enough ;) */
+	if(frompos == -1 && topos != -1)
+		frompos = 0;
+	if(topos < frompos) {
+		errmsg.LogError(0, RS_RET_ERR, "position.to=%d is lower than postion.from=%d\n",
+			topos, frompos);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
 
 	/* apply */
 	CHKmalloc(pTpe = tpeConstruct(pTpl));
@@ -1392,6 +1424,15 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	}
 	pTpe->data.field.fieldName = outname;
 	pTpe->data.field.eDateFormat = datefmt;
+	if(fieldnum != -1) {
+		pTpe->data.field.has_fields = 1;
+		pTpe->data.field.iFieldNr = fieldnum;
+		pTpe->data.field.field_delim = fielddelim;
+	}
+	if(frompos != -1) {
+		pTpe->data.field.iFromPos = frompos;
+		pTpe->data.field.iToPos = topos;
+	}
 finalize_it:
 	RETiRet;
 }
-- 
cgit 


From 00c4f69c559e5ba036e20a095843a1ca6eba57c8 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 19:08:07 +0200
Subject: milestone: LIST-type templates support full option set

---
 template.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 template.h | 10 ++++-----
 2 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/template.c b/template.c
index 7a4c398f..4cad7cb7 100644
--- a/template.c
+++ b/template.c
@@ -1243,12 +1243,17 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	int topos = -1;
 	int fieldnum = -1;
 	int fielddelim = 9; /* default is HT (USACSII 9) */
+	int re_matchToUse = 0;
+	int re_submatchToUse = 0;
+	char *re_expr = NULL;
 	struct cnfparamvals *pvals;
 	enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE;
 	enum {CC_NONE, CC_ESCAPE, CC_SPACE, CC_DROP} controlchr = CC_NONE;
 	enum {SP_NONE, SP_DROP, SP_REPLACE} secpath = SP_NONE;
 	enum tplFormatCaseConvTypes caseconv = tplCaseConvNo;
 	enum tplFormatTypes datefmt = tplFmtDefault;
+	enum tplRegexType re_type = TPL_REGEX_BRE;
+	enum tlpRegexNoMatchType re_nomatchType = TPL_REGEX_NOMATCH_USE_DFLTSTR;
 	DEFiRet;
 
 	/* pull params */
@@ -1277,6 +1282,40 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 			fieldnum = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "field.delimiter")) {
 			fielddelim = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.expression")) {
+			re_expr = es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.type")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"BRE", sizeof("BRE")-1)) {
+				re_type = TPL_REGEX_BRE;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"ERE", sizeof("ERE")-1)) {
+				re_type = TPL_REGEX_ERE;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid regex.type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.nomatchmode")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"DFLT", sizeof("DFLT")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_DFLTSTR;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"BLANK", sizeof("BLANK")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_BLANK;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"FIELD", sizeof("FIELD")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_WHOLE_FIELD;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"ZERO", sizeof("ZERO")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_ZERO;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid format type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.match")) {
+			re_matchToUse = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.submatch")) {
+			re_submatchToUse = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "format")) {
 			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"csv", sizeof("csv")-1)) {
 				formatType = F_CSV;
@@ -1369,6 +1408,11 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 			topos, frompos);
 		ABORT_FINALIZE(RS_RET_ERR);
 	}
+	if(fieldnum != -1 && re_expr != NULL) {
+		errmsg.LogError(0, RS_RET_ERR, "both field extraction and regex extraction "
+				"specified - this is not possible, remove one");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
 
 	/* apply */
 	CHKmalloc(pTpe = tpeConstruct(pTpl));
@@ -1433,6 +1477,34 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		pTpe->data.field.iFromPos = frompos;
 		pTpe->data.field.iToPos = topos;
 	}
+	if(re_expr != NULL) {
+		rsRetVal iRetLocal;
+		pTpe->data.field.typeRegex = re_type;
+		pTpe->data.field.nomatchAction = re_nomatchType;
+		pTpe->data.field.iMatchToUse = re_matchToUse;
+		pTpe->data.field.iSubMatchToUse = re_submatchToUse;
+		pTpe->data.field.has_regex = 1;
+		if((iRetLocal = objUse(regexp, LM_REGEXP_FILENAME)) == RS_RET_OK) {
+			int iOptions;
+			iOptions = (pTpe->data.field.typeRegex == TPL_REGEX_ERE) ? REG_EXTENDED : 0;
+			if(regexp.regcomp(&(pTpe->data.field.re), (char*) re_expr, iOptions) != 0) {
+				dbgprintf("error: can not compile regex: '%s'\n", re_expr);
+				errmsg.LogError(0, NO_ERRCODE, "error compiling regex '%s'", re_expr);
+				pTpe->data.field.has_regex = 2;
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else {
+			/* regexp object could not be loaded */
+			if(bFirstRegexpErrmsg) { /* prevent flood of messages, maybe even an endless loop! */
+				bFirstRegexpErrmsg = 0;
+				errmsg.LogError(0, NO_ERRCODE, "regexp library could not be loaded (error %d), "
+						"regexp ignored", iRetLocal);
+			}
+			pTpe->data.field.has_regex = 2;
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	}
+
 finalize_it:
 	RETiRet;
 }
diff --git a/template.h b/template.h
index c58d452c..65435cd8 100644
--- a/template.h
+++ b/template.h
@@ -58,6 +58,9 @@ enum tplFormatTypes { tplFmtDefault = 0, tplFmtMySQLDate = 1,
                       tplFmtRFC3164Date = 2, tplFmtRFC3339Date = 3, tplFmtPgSQLDate = 4,
 		      tplFmtSecFrac = 5, tplFmtRFC3164BuggyDate = 6, tplFmtUnixDate};
 enum tplFormatCaseConvTypes { tplCaseConvNo = 0, tplCaseConvUpper = 1, tplCaseConvLower = 2 };
+enum tplRegexType { TPL_REGEX_BRE = 0, /* posix BRE */
+		    TPL_REGEX_ERE = 1  /* posix ERE */
+		  };
 
 #include "msg.h"
 
@@ -80,11 +83,8 @@ struct templateEntry {
 			short has_regex;
 			short iMatchToUse;/* which match should be obtained (10 max) */
 			short iSubMatchToUse;/* which submatch should be obtained (10 max) */
-			enum {
-				TPL_REGEX_BRE = 0, /* posix BRE */
-				TPL_REGEX_ERE = 1  /* posix ERE */
-			} typeRegex;
-			enum {
+			enum tplRegexType typeRegex;
+			enum tlpRegexNoMatchType {
 				TPL_REGEX_NOMATCH_USE_DFLTSTR = 0, /* use the (old style) default "**NO MATCH**" string */
 				TPL_REGEX_NOMATCH_USE_BLANK = 1, /* use a blank string */
 				TPL_REGEX_NOMATCH_USE_WHOLE_FIELD = 2, /* use the full field contents that we were searching in*/
-- 
cgit 


From 3d56820f130e6c1b674560125e677be3b6a2d8f4 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 19:21:12 +0200
Subject: add capability to configure outname for constant (inside template)

also some cleanup
---
 grammar/grammar.y | 10 ++++------
 runtime/msg.c     |  4 ++--
 template.c        | 21 ++++++++-------------
 template.h        |  2 +-
 4 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index 27d0e1d5..8371f854 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -133,18 +133,16 @@ conf:	/* empty (to end recursion) */
 	| conf BSD_HOST_SELECTOR	{ cnfDoBSDHost($2); }
 obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
 	| BEGIN_ACTION nvlst ENDOBJ 	{ $$ = cnfobjNew(CNFOBJ_ACTION, $2); }
-        | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2);  dbgprintf("processing template() without {}\n"); }
+        | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2); }
         | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
 					{ $$ = cnfobjNew(CNFOBJ_TPL, $2);
 					  $$->subobjs = $5;
-					  dbgprintf("processing template() WITH {}, subobj=%p\n", $5); }
+					}
 propconst:				{ $$ = NULL; }
 	| propconst property		{ $$ = objlstAdd($1, $2); }
 	| propconst constant		{ $$ = objlstAdd($1, $2); }
-property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
-					  dbgprintf("processed property()\n"); }
-constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2);
-					  dbgprintf("processed constant()\n"); }
+property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2); }
+constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2); }
 cfsysline: CFSYSLINE	 		{ $$ = $1; }
 nvlst:					{ $$ = NULL; }
 	| nvlst nv 			{ $2->next = $1; $$ = $2; }
diff --git a/runtime/msg.c b/runtime/msg.c
index da751dba..891907ec 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2515,9 +2515,9 @@ jsonField(struct templateEntry *pTpe, uchar **ppRes, unsigned short *pbMustBeFre
 	pSrc = *ppRes;
 	buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen;
 	/* we hope we have only few escapes... */
-	dst = es_newStr(buflen+es_strlen(pTpe->data.field.fieldName)+15);
+	dst = es_newStr(buflen+es_strlen(pTpe->fieldName)+15);
 	es_addChar(&dst, '"');
-	es_addStr(&dst, pTpe->data.field.fieldName);
+	es_addStr(&dst, pTpe->fieldName);
 	es_addBufConstcstr(&dst, "\"=\"");
 	CHKiRet(jsonAddVal(pSrc, buflen, &dst));
 	es_addChar(&dst, '"');
diff --git a/template.c b/template.c
index 4cad7cb7..768608f1 100644
--- a/template.c
+++ b/template.c
@@ -74,7 +74,6 @@ static struct cnfparamdescr cnfparamdescrProperty[] = {
 	{ "controlcharacters", eCmdHdlrString, 0 },
 	{ "securepath", eCmdHdlrString, 0 },
 	{ "format", eCmdHdlrString, 0 },
-// From here
 	{ "position.from", eCmdHdlrInt, 0 },
 	{ "position.to", eCmdHdlrInt, 0 },
 	{ "field.number", eCmdHdlrInt, 0 },
@@ -84,7 +83,6 @@ static struct cnfparamdescr cnfparamdescrProperty[] = {
 	{ "regex.nomatchmode", eCmdHdlrString, 0 },
 	{ "regex.match", eCmdHdlrInt, 0 },
 	{ "regex.submatch", eCmdHdlrInt, 0 },
-//-- to here
 	{ "droplastlf", eCmdHdlrBinary, 0 },
 	{ "spifno1stsp", eCmdHdlrBinary, 0 }
 };
@@ -97,7 +95,6 @@ static struct cnfparamblk pblkProperty =
 static struct cnfparamdescr cnfparamdescrConstant[] = {
 	{ "value", eCmdHdlrString, 1 },
 	{ "outname", eCmdHdlrString, 0 },
-	{ "format", eCmdHdlrString, 0 }
 };
 static struct cnfparamblk pblkConstant =
 	{ CNFPARAMBLK_VERSION,
@@ -980,12 +977,12 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 
 	/* save field name - if none was given, use the property name instead */
 	if(pStrField == NULL) {
-		if((pTpe->data.field.fieldName =
+		if((pTpe->fieldName =
 		      es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp), cstrLen(pStrProp))) == NULL) {
 			return 1;
 		}
 	} else {
-		if((pTpe->data.field.fieldName =
+		if((pTpe->fieldName =
 		      es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrField), cstrLen(pStrField))) == NULL) {
 			return 1;
 		}
@@ -1193,6 +1190,7 @@ createConstantTpe(struct template *pTpl, struct cnfobj *o)
 	es_str_t *value;
 	int i;
 	struct cnfparamvals *pvals;
+	es_str_t *outname = NULL;
 	DEFiRet;
 
 	/* pull params */
@@ -1204,12 +1202,8 @@ createConstantTpe(struct template *pTpl, struct cnfobj *o)
 			continue;
 		if(!strcmp(pblkConstant.descr[i].name, "value")) {
 			value = pvals[i].val.d.estr;
-		} else if(!strcmp(pblkConstant.descr[i].name, "format")) {
-			errmsg.LogError(0, RS_RET_ERR, "paramter 'format' is currently not "
-				"supported for 'constant' template parts - ignored");
 		} else if(!strcmp(pblkConstant.descr[i].name, "outname")) {
-			errmsg.LogError(0, RS_RET_ERR, "paramter 'outname' is currently not "
-				"supported for 'constant' template parts - ignored");
+			outname = es_strdup(pvals[i].val.d.estr);
 		} else {
 			dbgprintf("template:constantTpe: program error, non-handled "
 			  "param '%s'\n", pblkConstant.descr[i].name);
@@ -1222,6 +1216,7 @@ createConstantTpe(struct template *pTpl, struct cnfobj *o)
 	CHKmalloc(pTpe = tpeConstruct(pTpl));
 	es_unescapeStr(value);
 	pTpe->eEntryType = CONSTANT;
+	pTpe->fieldName = outname;
 	pTpe->data.constant.iLenConstant = es_strlen(value);
 	pTpe->data.constant.pConstant = (uchar*)es_str2cstr(value, NULL);
 
@@ -1466,7 +1461,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		pTpe->data.field.options.bSecPathReplace = 1;
 		break;
 	}
-	pTpe->data.field.fieldName = outname;
+	pTpe->fieldName = outname;
 	pTpe->data.field.eDateFormat = datefmt;
 	if(fieldnum != -1) {
 		pTpe->data.field.has_fields = 1;
@@ -1767,8 +1762,8 @@ void tplDeleteAll(rsconf_t *conf)
 				}
 				if(pTpeDel->data.field.propName != NULL)
 					es_deleteStr(pTpeDel->data.field.propName);
-				if(pTpeDel->data.field.fieldName != NULL)
-					es_deleteStr(pTpeDel->data.field.fieldName);
+				if(pTpeDel->fieldName != NULL)
+					es_deleteStr(pTpeDel->fieldName);
 #endif
 				break;
 			}
diff --git a/template.h b/template.h
index 65435cd8..9f6a4c33 100644
--- a/template.h
+++ b/template.h
@@ -68,6 +68,7 @@ enum tplRegexType { TPL_REGEX_BRE = 0, /* posix BRE */
 struct templateEntry {
 	struct templateEntry *pNext;
 	enum EntryTypes eEntryType;
+	es_str_t *fieldName;	/**< field name to be used for structured output */
 	union {
 		struct {
 			uchar *pConstant;	/* pointer to constant value */
@@ -99,7 +100,6 @@ struct templateEntry {
 #endif
 
 			es_str_t *propName;	/**< property name (currently being used for CEE only) */
-			es_str_t *fieldName;	/**< field name to be used for structured output */
 
 			enum tplFormatTypes eDateFormat;
 			enum tplFormatCaseConvTypes eCaseConv;
-- 
cgit