introduced the idea of detached properties

some things inside the message can be used over a large number of
messages and need to to be allocated and re-written every time. I now
begin to implement this as a "prop_t" object, first use for the inputName.
Some input modules are already converted, some others to go. Will do
a little performance check on the new method before I go further.

Also, this commit has some cleanup and a few bug fixes that prevented
compiliation in debug mode (I overlooked this as I did not compile
for debug, what I normally do, and the automatted test also does not
do that)

9 files changed, 152 insertions, 37 deletions

diff --git a/runtime/datetime.c b/runtime/datetime.c
index e0f3f5fa..ea67eec5 100644
--- a/runtime/datetime.c
+++ b/runtime/datetime.c
@@ -556,7 +556,6 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf)
 	 */
 	assert(ts != NULL);
 	assert(pBuf != NULL);
-	assert(iLenDst >= 15);
 
 	pBuf[0] = (ts->year / 1000) % 10 + '0';
 	pBuf[1] = (ts->year / 100) % 10 + '0';
@@ -582,7 +581,6 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf)
 	/* see note in formatTimestampToMySQL, applies here as well */
 	assert(ts != NULL);
 	assert(pBuf != NULL);
-	assert(iLenDst >= 20);
 
 	pBuf[0] = (ts->year / 1000) % 10 + '0';
 	pBuf[1] = (ts->year / 100) % 10 + '0';
@@ -665,7 +663,6 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf)
 	BEGINfunc
 	assert(ts != NULL);
 	assert(pBuf != NULL);
-	assert(iLenBuf >= 33);
 
 	/* start with fixed parts */
 	/* year yyyy */
@@ -740,8 +737,6 @@ int formatTimestamp3164(struct syslogTime *ts, char* pBuf)
 	assert(ts != NULL);
 	assert(pBuf != NULL);
 	
-	assert(iLenBuf >= 16);
-
 	pBuf[0] = monthNames[(ts->month - 1)% 12][0];
 	pBuf[1] = monthNames[(ts->month - 1) % 12][1];
 	pBuf[2] = monthNames[(ts->month - 1) % 12][2];
diff --git a/runtime/datetime.h b/runtime/datetime.h
index 79a86d05..58f368e7 100644
--- a/runtime/datetime.h
+++ b/runtime/datetime.h
@@ -23,8 +23,6 @@
 #ifndef INCLUDED_DATETIME_H
 #define INCLUDED_DATETIME_H
 
-#include "datetime.h"
-
 /* TODO: define error codes */
 #define NO_ERRCODE -1
 
diff --git a/runtime/msg.c b/runtime/msg.c
index c4ba18df..47b4ed85 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -48,6 +48,7 @@
 #include "atomic.h"
 #include "unicode-helper.h"
 #include "ruleset.h"
+#include "prop.h"
 
 /* static data */
 DEFobjStaticHelpers
@@ -55,6 +56,7 @@ DEFobjCurrIf(var)
 DEFobjCurrIf(datetime)
 DEFobjCurrIf(glbl)
 DEFobjCurrIf(regexp)
+DEFobjCurrIf(prop)
 
 static struct {
 	uchar *pszName;
@@ -273,12 +275,26 @@ static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7",
 /* some forward declarations */
 static int getAPPNAMELen(msg_t *pM, bool bLockMutex);
 
+
 static inline int getProtocolVersion(msg_t *pM)
 {
 	return(pM->iProtocolVersion);
 }
 
 
+static inline void
+getInputName(msg_t *pM, uchar **ppsz, int *plen)
+{
+	BEGINfunc
+	if(pM == NULL) {
+		*ppsz = UCHAR_CONSTANT("");
+		*plen = 0;
+	} else {
+		prop.GetString(pM->pInputName, ppsz, plen);
+	}
+	ENDfunc
+}
+
 /* The following functions will support advanced output module
  * multithreading, once this is implemented. Currently, we
  * include them as hooks only. The idea is that we need to guard
@@ -431,7 +447,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 	pM->iFacility = -1;
 	pM->offAfterPRI = 0;
 	pM->offMSG = -1;
-	pM->iLenInputName = 0;
 	pM->iProtocolVersion = 0;
 	pM->msgFlags = 0;
 	pM->iLenRawMsg = 0;
@@ -444,7 +459,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 	pM->pszHOSTNAME = NULL;
 	pM->pszRcvFrom = NULL;
 	pM->pszRcvFromIP = NULL;
-	pM->pszInputName = NULL;
 	pM->pszRcvdAt3164 = NULL;
 	pM->pszRcvdAt3339 = NULL;
 	pM->pszRcvdAt_MySQL = NULL;
@@ -458,6 +472,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 	pM->pCSAPPNAME = NULL;
 	pM->pCSPROCID = NULL;
 	pM->pCSMSGID = NULL;
+	pM->pInputName = NULL;
 	pM->pRuleset = NULL;
 	memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt));
 	memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP));
@@ -556,7 +571,8 @@ CODESTARTobjDestruct(msg)
 			free(pThis->pszRawMsg);
 		freeTAG(pThis);
 		freeHOSTNAME(pThis);
-		free(pThis->pszInputName);
+		if(pThis->pInputName != NULL)
+			prop.Destruct(&pThis->pInputName);
 		free(pThis->pszRcvFrom);
 		free(pThis->pszRcvFromIP);
 		free(pThis->pszRcvdAt3164);
@@ -719,11 +735,16 @@ msg_t* MsgDup(msg_t* pOld)
  */
 static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm)
 {
+	uchar *psz;
+	int len;
 	DEFiRet;
 
 	assert(pThis != NULL);
 	assert(pStrm != NULL);
 
+	/* "pump" some property values into strings */
+
+	/* then serialize elements */
 	CHKiRet(obj.BeginSerialize(pStrm, (obj_t*) pThis));
 	objSerializeSCALAR(pStrm, iProtocolVersion, SHORT);
 	objSerializeSCALAR(pStrm, iSeverity, SHORT);
@@ -743,7 +764,9 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm)
 
 	objSerializePTR(pStrm, pszRawMsg, PSZ);
 	objSerializePTR(pStrm, pszHOSTNAME, PSZ);
-	objSerializePTR(pStrm, pszInputName, PSZ);
+	getInputName(pThis, &psz, &len);
+	objSerializeSCALAR_VAR(pStrm, "pszInputName", PSZ, psz); 
+	//objSerializePTR(pStrm, pszInputName, PSZ);
 	objSerializePTR(pStrm, pszRcvFrom, PSZ);
 	objSerializePTR(pStrm, pszRcvFromIP, PSZ);
 
@@ -1427,18 +1450,6 @@ char *getHOSTNAME(msg_t *pM)
 }
 
 
-static uchar *getInputName(msg_t *pM)
-{
-	if(pM == NULL)
-		return (uchar*) "";
-	else
-		if(pM->pszInputName == NULL)
-			return (uchar*) "";
-		else
-			return pM->pszInputName;
-}
-
-
 uchar *getRcvFrom(msg_t *pM)
 {
 	if(pM == NULL)
@@ -1598,14 +1609,31 @@ static int getAPPNAMELen(msg_t *pM, bool bLockMutex)
 /* rgerhards 2008-09-10: set pszInputName in msg object
  * rgerhards, 2009-06-16
  */
-void MsgSetInputName(msg_t *pMsg, uchar* pszInputName, size_t lenInputName)
+void MsgSetInputName(msg_t *pThis, prop_t *inputName)
 {
-	assert(pMsg != NULL);
-	free(pMsg->pszInputName);
-	pMsg->iLenInputName = lenInputName;
-	if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) {
-		memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1);
-	}
+	assert(pThis != NULL);
+
+	if(pThis->pInputName != NULL)
+		prop.Destruct(&pThis->pInputName);
+	pThis->pInputName = inputName;
+}
+
+/* to be removed soon: work-around for those tht can not natively generate an
+ * input name.
+ * rgerhards, 2009-06-29
+ */
+void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len)
+{
+	prop_t *pProp;
+	assert(pThis != NULL);
+
+	/* we need to create a property */ 
+	prop.Construct(&pProp);
+	prop.SetString(pProp, psz, len);
+	prop.ConstructFinalize(pProp);
+	prop.AddRef(pProp);
+	MsgSetInputName(pThis, pProp);
+	prop.Destruct(&pProp);
 }
 
 /* rgerhards 2004-11-16: set pszRcvFrom in msg object
@@ -1875,6 +1903,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 	short iOffs;
 
 	BEGINfunc
+dbgprintf("XXXX: msgGetProp for %d\n", propID);
 	assert(pMsg != NULL);
 	assert(pbMustBeFreed != NULL);
 
@@ -1910,7 +1939,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 			break;
 		*/
 		case PROP_INPUTNAME:
-			pRes = (char*) getInputName(pMsg);
+RUNLOG;
+			getInputName(pMsg, ((uchar**) &pRes), &bufLen);
+RUNLOG;
+RUNLOG_VAR("%p", pRes);
 			break;
 		case PROP_FROMHOST:
 			pRes = (char*) getRcvFrom(pMsg);
@@ -2738,6 +2770,7 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID)
 #define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1)
 rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp)
 {
+	prop_t *myProp;
 	DEFiRet;
 
 	ISOBJ_TYPE_assert(pThis, msg);
@@ -2765,7 +2798,13 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp)
 	} else if(isProp("pszTAG")) {
 		MsgSetTAG(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr));
 	} else if(isProp("pszInputName")) {
-		MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr));
+		/* we need to create a property */ 
+		CHKiRet(prop.Construct(&myProp));
+		CHKiRet(prop.SetString(myProp, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)));
+		CHKiRet(prop.ConstructFinalize(myProp));
+		prop.AddRef(myProp);
+		MsgSetInputName(pThis, myProp);
+		prop.Destruct(&myProp);
 	} else if(isProp("pszRcvFromIP")) {
 		MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr));
 	} else if(isProp("pszRcvFrom")) {
@@ -2790,6 +2829,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp)
 		dbgprintf("no longer supported property pszMSG silently ignored\n");
 	}
 
+finalize_it:
 	RETiRet;
 }
 #undef	isProp
@@ -2833,6 +2873,7 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE)
 	CHKiRet(objUse(var, CORE_COMPONENT));
 	CHKiRet(objUse(datetime, CORE_COMPONENT));
 	CHKiRet(objUse(glbl, CORE_COMPONENT));
+	CHKiRet(objUse(prop, CORE_COMPONENT));
 
 	/* set our own handlers */
 	OBJSetMethodHandler(objMethod_SERIALIZE, MsgSerialize);
diff --git a/runtime/msg.h b/runtime/msg.h
index e6b25e6c..a1fc535b 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -72,7 +72,6 @@ struct msg {
 	short	iFacility;	/* Facility code 0 .. 23*/
 	short	offAfterPRI;	/* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */
 	short	offMSG;		/* offset at which the MSG part starts in pszRawMsg */
-	short	iLenInputName;	/* Length of pszInputName */
 	short	iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */
 	int	msgFlags;	/* flags associated with this message */
 	int	iLenRawMsg;	/* length of raw message */
@@ -86,7 +85,6 @@ struct msg {
 	uchar	*pszHOSTNAME;	/* HOSTNAME from syslog message */
 	uchar	*pszRcvFrom;	/* System message was received from */
 	uchar	*pszRcvFromIP;	/* IP of system message was received from */
-	uchar *pszInputName;	/* name of the input module that submitted this message */
 	char *pszRcvdAt3164;	/* time as RFC3164 formatted string (always 15 charcters) */
 	char *pszRcvdAt3339;	/* time as RFC3164 formatted string (32 charcters at most) */
 	char *pszRcvdAt_MySQL;	/* rcvdAt as MySQL formatted string (always 14 charcters) */
@@ -100,6 +98,7 @@ struct msg {
 	cstr_t *pCSAPPNAME;	/* APP-NAME */
 	cstr_t *pCSPROCID;	/* PROCID */
 	cstr_t *pCSMSGID;	/* MSGID */
+	prop_t *pInputName;	/* input name property */
 	ruleset_t *pRuleset;	/* ruleset to be used for processing this message */
 	time_t ttGenTime;	/* time msg object was generated, same as tRcvdAt, but a Unix timestamp.
 				   While this field looks redundant, it is required because a Unix timestamp
@@ -144,7 +143,7 @@ rsRetVal msgDestruct(msg_t **ppM);
 msg_t* MsgDup(msg_t* pOld);
 msg_t *MsgAddRef(msg_t *pM);
 void setProtocolVersion(msg_t *pM, int iNewVersion);
-void MsgSetInputName(msg_t *pMsg, uchar*, size_t);
+void MsgSetInputName(msg_t *pMsg, prop_t*);
 rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME);
 rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID);
 rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID);
@@ -166,6 +165,12 @@ char *textpri(char *pRes, size_t pResLen, int pri);
 rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar);
 rsRetVal MsgEnableThreadSafety(void);
 
+
+// REMOVE:
+void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len);
+
+
+
 /* TODO: remove these five (so far used in action.c) */
 char *getMSG(msg_t *pM);
 char *getHOSTNAME(msg_t *pM);
diff --git a/runtime/prop.c b/runtime/prop.c
index 02be315f..989657dd 100644
--- a/runtime/prop.c
+++ b/runtime/prop.c
@@ -36,9 +36,12 @@
 #include "config.h"
 #include <stdlib.h>
 #include <assert.h>
+#include <string.h>
 
 #include "rsyslog.h"
 #include "obj.h"
+#include "obj-types.h"
+#include "atomic.h"
 #include "prop.h"
 
 /* static data */
@@ -48,8 +51,45 @@ DEFobjStaticHelpers
 /* Standard-Constructor
  */
 BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! */
+	pThis->iRefCount = 1;
 ENDobjConstruct(prop)
 
+/* set string, we make our own private copy! This MUST only be called BEFORE
+ * ConstructFinalize()!
+ */
+static rsRetVal SetString(prop_t *pThis, uchar *psz, int len)
+{
+	DEFiRet;
+	ISOBJ_TYPE_assert(pThis, prop);
+	if(pThis->len >= CONF_PROP_BUFSIZE)
+		free(pThis->szVal.psz);
+	pThis->len = len;
+	if(len < CONF_PROP_BUFSIZE) {
+		memcpy(pThis->szVal.sz, psz, len + 1);
+	} else {
+		CHKmalloc(pThis->szVal.psz = malloc(len + 1));
+		memcpy(pThis->szVal.sz, psz, len + 1);
+	}
+
+finalize_it:
+	RETiRet;
+}
+
+
+/* get string */
+static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen)
+{
+	BEGINfunc
+	ISOBJ_TYPE_assert(pThis, prop);
+	if(pThis->len < CONF_PROP_BUFSIZE)
+		*ppsz = pThis->szVal.sz;
+	else
+		*ppsz = pThis->szVal.psz;
+	*plen = pThis->len;
+	ENDfunc
+	return RS_RET_OK;
+}
+
 
 /* ConstructionFinalizer
  * rgerhards, 2008-01-09
@@ -63,9 +103,29 @@ propConstructFinalize(prop_t __attribute__((unused)) *pThis)
 }
 
 
+/* add a new reference. It is VERY IMPORTANT to call this function whenever
+ * the property is handed over to some entitiy that later call Destruct() on it.
+ */
+static rsRetVal AddRef(prop_t *pThis)
+{
+	ATOMIC_INC(pThis->iRefCount);
+	return RS_RET_OK;
+}
+
+
 /* destructor for the prop object */
 BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */
+	int currRefCount;
 CODESTARTobjDestruct(prop)
+	currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount);
+	if(currRefCount == 0) {
+		/* (only) in this case we need to actually destruct the object */
+dbgprintf("XXXXX: propDestruct: ptr %p, pThis %p, len %d\n", pThis->szVal.psz, pThis, pThis->len);
+		if(pThis->len >= CONF_PROP_BUFSIZE)
+			free(pThis->szVal.psz);
+	} else {
+		pThis = NULL; /* tell framework NOT to destructing the object! */
+	}
 ENDobjDestruct(prop)
 
 
@@ -94,6 +154,9 @@ CODESTARTobjQueryInterface(prop)
 	pIf->ConstructFinalize = propConstructFinalize;
 	pIf->Destruct = propDestruct;
 	pIf->DebugPrint = propDebugPrint;
+	pIf->SetString = SetString;
+	pIf->GetString = GetString;
+	pIf->AddRef = AddRef;
 
 finalize_it:
 ENDobjQueryInterface(prop)
diff --git a/runtime/prop.h b/runtime/prop.h
index 7fc466b5..1d18c650 100644
--- a/runtime/prop.h
+++ b/runtime/prop.h
@@ -28,14 +28,23 @@
 /* the prop object */
 struct prop_s {
 	BEGINobjInstance;	/* Data to implement generic object - MUST be the first data element! */
+	int iRefCount;		/* reference counter */
+	union { 
+		uchar *psz;		/* stored string */
+		uchar sz[CONF_PROP_BUFSIZE];
+	} szVal;
+	int len;		/* we use int intentionally, otherwise we may get some troubles... */
 };
 
 /* interfaces */
 BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */
 	INTERFACEObjDebugPrint(prop);
 	rsRetVal (*Construct)(prop_t **ppThis);
-	rsRetVal (*ConstructFinalize)(prop_t __attribute__((unused)) *pThis);
+	rsRetVal (*ConstructFinalize)(prop_t *pThis);
 	rsRetVal (*Destruct)(prop_t **ppThis);
+	rsRetVal (*SetString)(prop_t *pThis, uchar* psz, int len);
+	rsRetVal (*GetString)(prop_t *pThis, uchar** ppsz, int *plen);
+	rsRetVal (*AddRef)(prop_t *pThis);
 ENDinterface(prop)
 #define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
 
diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c
index 6f732f0e..faa74427 100644
--- a/runtime/rsyslog.c
+++ b/runtime/rsyslog.c
@@ -77,6 +77,7 @@
 #include "conf.h"
 #include "glbl.h"
 #include "errmsg.h"
+#include "prop.h"
 #include "rule.h"
 #include "ruleset.h"
 
@@ -150,6 +151,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF)
 		CHKiRet(glblClassInit(NULL));
 		if(ppErrObj != NULL) *ppErrObj = "datetime";
 		CHKiRet(datetimeClassInit(NULL));
+		if(ppErrObj != NULL) *ppErrObj = "prop";
+		CHKiRet(propClassInit(NULL));
 		if(ppErrObj != NULL) *ppErrObj = "msg";
 		CHKiRet(msgClassInit(NULL));
 		if(ppErrObj != NULL) *ppErrObj = "ctok_token";
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 4e0f6e6c..8a043dde 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -39,6 +39,7 @@
 #define CONF_RAWMSG_BUFSIZE		101
 #define CONF_TAG_BUFSIZE		32
 #define CONF_HOSTNAME_BUFSIZE		32
+#define CONF_PROP_BUFSIZE		16	/* should be close to sizeof(ptr) or lighly above it */
 
 
 /* ############################################################# *
diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h
index 400c74eb..c5130238 100644
--- a/runtime/stringbuf.h
+++ b/runtime/stringbuf.h
@@ -35,6 +35,7 @@
 #ifndef _STRINGBUF_H_INCLUDED__
 #define _STRINGBUF_H_INCLUDED__ 1
 
+#include <assert.h>
 
 /** 
  * The dynamic string buffer object.
@@ -48,7 +49,6 @@ typedef struct cstr_s
 	uchar *pszBuf;		/**< pointer to the sz version of the string (after it has been created )*/
 	size_t iBufSize;	/**< current maximum size of the string buffer */
 	size_t iStrLen;		/**< length of the string in characters. */
-	bool bIsForeignBuf;	/**< is pBuf a buffer provided by someone else? */
 } cstr_t;