moved DNS resolution code out of imudp and into the backend processing

Most importantly, DNS resolution now never happens if the resolved name
is not required. Note that this applies to imudp - for the other inputs,
DNS resolution almost comes for free, so we do not do it there. However,
the new method has been implemented in a generic way and as such may
also be used by other modules in the future.

6 files changed, 171 insertions, 37 deletions

diff --git a/runtime/msg.c b/runtime/msg.c
index 623c5b4a..0597ea84 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -35,6 +35,8 @@
 #include <string.h>
 #include <assert.h>
 #include <ctype.h>
+#include <sys/socket.h>
+#include <netdb.h>
 #if HAVE_MALLOC_H
 #  include <malloc.h>
 #endif
@@ -51,6 +53,7 @@
 #include "unicode-helper.h"
 #include "ruleset.h"
 #include "prop.h"
+#include "net.h"
 
 /* static data */
 DEFobjStaticHelpers
@@ -59,6 +62,7 @@ DEFobjCurrIf(datetime)
 DEFobjCurrIf(glbl)
 DEFobjCurrIf(regexp)
 DEFobjCurrIf(prop)
+DEFobjCurrIf(net)
 
 static struct {
 	uchar *pszName;
@@ -284,6 +288,41 @@ static inline int getProtocolVersion(msg_t *pM)
 }
 
 
+/* do a DNS reverse resolution, if not already done, reflect status
+ * rgerhards, 2009-11-16
+ */
+static inline rsRetVal
+resolveDNS(msg_t *pMsg) {
+	rsRetVal localRet;
+	prop_t *propFromHost = NULL;
+	prop_t *propFromHostIP = NULL;
+	uchar fromHost[NI_MAXHOST];
+	uchar fromHostIP[NI_MAXHOST];
+	uchar fromHostFQDN[NI_MAXHOST];
+	DEFiRet;
+
+	CHKiRet(objUse(net, CORE_COMPONENT));
+	if(pMsg->msgFlags & NEEDS_DNSRESOL) {
+		localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP);
+		if(localRet == RS_RET_OK) {
+			MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost);
+			CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP));
+		}
+	}
+finalize_it:
+	if(iRet != RS_RET_OK) {
+		/* best we can do: remove property */
+		MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost);
+		prop.Destruct(&propFromHost);
+	}
+	if(propFromHost != NULL)
+		prop.Destruct(&propFromHost);
+	if(propFromHostIP != NULL)
+		prop.Destruct(&propFromHostIP);
+	RETiRet;
+}
+
+
 static inline void
 getInputName(msg_t *pM, uchar **ppsz, int *plen)
 {
@@ -307,6 +346,7 @@ getRcvFromIP(msg_t *pM)
 	if(pM == NULL) {
 		psz = UCHAR_CONSTANT("");
 	} else {
+		resolveDNS(pM); /* make sure we have a resolved entry */
 		if(pM->pRcvFromIP == NULL)
 			psz = UCHAR_CONSTANT("");
 		else
@@ -660,7 +700,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 	pM->pCSMSGID = NULL;
 	pM->pInputName = NULL;
 	pM->pRcvFromIP = NULL;
-	pM->pRcvFrom = NULL;
+	pM->rcvFrom.pRcvFrom = NULL;
 	pM->pRuleset = NULL;
 	memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt));
 	memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP));
@@ -761,8 +801,12 @@ CODESTARTobjDestruct(msg)
 		freeHOSTNAME(pThis);
 		if(pThis->pInputName != NULL)
 			prop.Destruct(&pThis->pInputName);
-		if(pThis->pRcvFrom != NULL)
-			prop.Destruct(&pThis->pRcvFrom);
+		if((pThis->msgFlags & NEEDS_DNSRESOL) == 0) {
+			if(pThis->rcvFrom.pRcvFrom != NULL)
+				prop.Destruct(&pThis->rcvFrom.pRcvFrom);
+		} else {
+			free(pThis->rcvFrom.pfrominet);
+		}
 		if(pThis->pRcvFromIP != NULL)
 			prop.Destruct(&pThis->pRcvFromIP);
 		free(pThis->pszRcvdAt3164);
@@ -848,6 +892,7 @@ ENDobjDestruct(msg)
 msg_t* MsgDup(msg_t* pOld)
 {
 	msg_t* pNew;
+	rsRetVal localRet;
 
 	assert(pOld != NULL);
 
@@ -868,9 +913,19 @@ msg_t* MsgDup(msg_t* pOld)
 	pNew->iLenMSG = pOld->iLenMSG;
 	pNew->iLenTAG = pOld->iLenTAG;
 	pNew->iLenHOSTNAME = pOld->iLenHOSTNAME;
-	if(pOld->pRcvFrom != NULL) {
-		pNew->pRcvFrom = pOld->pRcvFrom;
-		prop.AddRef(pNew->pRcvFrom);
+	if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) {
+			localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet);
+			if(localRet != RS_RET_OK) {
+				/* if something fails, we accept loss of this property, it is
+				 * better than losing the whole message.
+				 */
+				pNew->msgFlags &= ~NEEDS_DNSRESOL;
+			}
+	} else {
+		if(pOld->rcvFrom.pRcvFrom != NULL) {
+			pNew->rcvFrom.pRcvFrom = pOld->rcvFrom.pRcvFrom;
+			prop.AddRef(pNew->rcvFrom.pRcvFrom);
+		}
 	}
 	if(pOld->pRcvFromIP != NULL) {
 		pNew->pRcvFromIP = pOld->pRcvFromIP;
@@ -1648,12 +1703,13 @@ int getHOSTNAMELen(msg_t *pM)
 	if(pM == NULL)
 		return 0;
 	else
-		if(pM->pszHOSTNAME == NULL)
-			if(pM->pRcvFrom == NULL)
+		if(pM->pszHOSTNAME == NULL) {
+			resolveDNS(pM);
+			if(pM->rcvFrom.pRcvFrom == NULL)
 				return 0;
 			else
-				return prop.GetStringLen(pM->pRcvFrom);
-		else
+				return prop.GetStringLen(pM->rcvFrom.pRcvFrom);
+		} else
 			return pM->iLenHOSTNAME;
 }
 
@@ -1664,12 +1720,13 @@ char *getHOSTNAME(msg_t *pM)
 		return "";
 	else
 		if(pM->pszHOSTNAME == NULL) {
-			if(pM->pRcvFrom == NULL) {
+			resolveDNS(pM);
+			if(pM->rcvFrom.pRcvFrom == NULL) {
 				return "";
 			} else {
 				uchar *psz;
 				int len;
-				prop.GetString(pM->pRcvFrom, &psz, &len);
+				prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len);
 				return (char*) psz;
 			}
 		} else {
@@ -1683,13 +1740,15 @@ uchar *getRcvFrom(msg_t *pM)
 	uchar *psz;
 	int len;
 	BEGINfunc
+
 	if(pM == NULL) {
 		psz = UCHAR_CONSTANT("");
 	} else {
-		if(pM->pRcvFrom == NULL)
+		resolveDNS(pM);
+		if(pM->rcvFrom.pRcvFrom == NULL)
 			psz = UCHAR_CONSTANT("");
 		else
-			prop.GetString(pM->pRcvFrom, &psz, &len);
+			prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len);
 	}
 	ENDfunc
 	return psz;
@@ -1845,6 +1904,28 @@ void MsgSetInputName(msg_t *pThis, prop_t *inputName)
 }
 
 
+/* Set the pfrominet socket store, so that we can obtain the peer at some
+ * later time. Note that we do not check if pRcvFrom is already set, so this
+ * function must only be called during message creation.
+ * NOTE: msgFlags is NOT set. While this is somewhat a violation of layers,
+ * it is done because it gains us some performance. So the caller must make
+ * sure the message flags are properly maintained. For all current callers,
+ * this is always the case and without extra effort required.
+ * rgerhards, 2009-11-17
+ */
+rsRetVal
+msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa){ 
+	DEFiRet;
+	assert(pThis->rcvFrom.pRcvFrom == NULL);
+
+	CHKmalloc(pThis->rcvFrom.pfrominet = malloc(sizeof(struct sockaddr_storage)));
+	memcpy(pThis->rcvFrom.pfrominet, sa, sizeof(struct sockaddr_storage));
+
+finalize_it:
+	RETiRet;
+}
+
+
 /* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef()
  * on the property, because this must be done in all current cases and there
  * is no case expected where this may not be necessary.
@@ -1855,9 +1936,15 @@ void MsgSetRcvFrom(msg_t *pThis, prop_t *new)
 	assert(pThis != NULL);
 
 	prop.AddRef(new);
-	if(pThis->pRcvFrom != NULL)
-		prop.Destruct(&pThis->pRcvFrom);
-	pThis->pRcvFrom = new;
+	if(pThis->msgFlags & NEEDS_DNSRESOL) {
+		if(pThis->rcvFrom.pfrominet != NULL)
+		free(pThis->rcvFrom.pfrominet);
+		pThis->msgFlags &= ~NEEDS_DNSRESOL;
+	} else {
+		if(pThis->rcvFrom.pRcvFrom != NULL)
+			prop.Destruct(&pThis->rcvFrom.pRcvFrom);
+	}
+	pThis->rcvFrom.pRcvFrom = new;
 }
 
 
diff --git a/runtime/msg.h b/runtime/msg.h
index 9101cef7..366dce64 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -61,12 +61,6 @@ struct msg {
 	pthread_mutex_t mut;
 	bool	bDoLock;	 /* use the mutex? */
 	short	iRefCount;	/* reference counter (0 = unused) */
-	   /* background: the hostname is not present on "regular" messages
-	    * received via UNIX domain sockets from the same machine. However,
-	    * it is available when we have a forwarder (e.g. rfc3195d) using local
-	    * sockets. All in all, the parser would need parse templates, that would
-	    * resolve all these issues... rgerhards, 2005-10-06
-	    */
 	short	iSeverity;	/* the severity 0..7 */
 	short	iFacility;	/* Facility code 0 .. 23*/
 	short	offAfterPRI;	/* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */
@@ -94,8 +88,12 @@ struct msg {
 	cstr_t *pCSPROCID;	/* PROCID */
 	cstr_t *pCSMSGID;	/* MSGID */
 	prop_t *pInputName;	/* input name property */
-	prop_t *pRcvFrom;	/* name of system message was received from */
 	prop_t *pRcvFromIP;	/* IP of system message was received from */
+	union {
+		prop_t *pRcvFrom;/* name of system message was received from */
+		struct sockaddr_storage *pfrominet; /* unresolved name */
+	} rcvFrom;
+
 	ruleset_t *pRuleset;	/* ruleset to be used for processing this message */
 	time_t ttGenTime;	/* time msg object was generated, same as tRcvdAt, but a Unix timestamp.
 				   While this field looks redundant, it is required because a Unix timestamp
@@ -129,6 +127,8 @@ struct msg {
 #define MARK		0x008	/* this message is a mark */
 #define NEEDS_PARSING	0x010	/* raw message, must be parsed before processing can be done */
 #define PARSE_HOSTNAME	0x020	/* parse the hostname during message parsing */
+#define NEEDS_DNSRESOL	0x040	/* fromhost address is unresolved and must be locked up via DNS reverse lookup first */
+#define NEEDS_ACLCHK_U	0x080	/* check UDP ACLs after DNS resolution has been done in main queue consumer */
 
 
 /* function prototypes
@@ -148,6 +148,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf);
 void MsgSetRuleset(msg_t *pMsg, ruleset_t*);
 rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl);
 rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData);
+rsRetVal msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa);
 void MsgSetRcvFrom(msg_t *pMsg, prop_t*);
 void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **);
 rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*);
diff --git a/runtime/net.c b/runtime/net.c
index dfae53e2..51db606a 100644
--- a/runtime/net.c
+++ b/runtime/net.c
@@ -892,15 +892,18 @@ rsRetVal addAllowedSenderLine(char* pName, uchar** ppRestOfConfLine)
  * including IPv4/v6 as well as domain name wildcards.
  * This is a helper to isAllowedSender. As it is only called once, it is
  * declared inline.
- * Returns 0 if they do not match, something else otherwise.
- * contributed 1007-07-16 by mildew@gmail.com
+ * Returns 0 if they do not match, 1 if they match and 2 if a DNS name would have been required.
+ * contributed 2007-07-16 by mildew@gmail.com
  */
-static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost)
+static inline int
+MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS)
 {
 	assert(pAllow != NULL);
 	assert(pFrom != NULL);
 
 	if(F_ISSET(pAllow->flags, ADDR_NAME)) {
+		if(bChkDNS == 0)
+			return 2;
 		dbgprintf("MaskCmp: host=\"%s\"; pattern=\"%s\"\n", pszFromHost, pAllow->addr.HostWildcard);
 		
 #		if !defined(FNM_CASEFOLD)
@@ -967,18 +970,22 @@ static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr
 /* check if a sender is allowed. The root of the the allowed sender.
  * list must be proveded by the caller. As such, this function can be
  * used to check both UDP and TCP allowed sender lists.
- * returns 1, if the sender is allowed, 0 otherwise.
+ * returns 1, if the sender is allowed, 0 if not and 2 if we could not 
+ * obtain a result because we would need a dns name, which we don't have
+ * (2 was added rgerhards, 2009-11-16).
  * rgerhards, 2005-09-26
  */
-static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost)
+static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS)
 {
 	struct AllowedSenders *pAllow;
 	struct AllowedSenders *pAllowRoot;
+	int bNeededDNS = 0;	/* partial check because we could not resolve DNS? */
+	int ret;
 
 	assert(pFrom != NULL);
 	
 	if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK)
-		return 0;	/* if something went wrong, we denie access - that's the better choice... */
+		return 0;	/* if something went wrong, we deny access - that's the better choice... */
 
 	if(pAllowRoot == NULL)
 		return 1; /* checking disabled, everything is valid! */
@@ -990,10 +997,20 @@ static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *p
 	 * that the sender is disallowed.
 	 */
 	for(pAllow = pAllowRoot ; pAllow != NULL ; pAllow = pAllow->pNext) {
-		if (MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost))
+		ret = MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost, bChkDNS);
+		if(ret == 1)
 			return 1;
+		else if(ret == 2)
+			bNeededDNS = 2;
 	}
-	return 0;
+	return bNeededDNS;
+}
+
+
+/* legacy API, not to be used any longer */
+static int
+isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) {
+	return isAllowedSender2(pszType, pFrom, pszFromHost, 1);
 }
 
 
@@ -1533,12 +1550,36 @@ static int CmpHost(struct sockaddr_storage *s1, struct sockaddr_storage* s2, siz
 		ret = memcmp(s1, s2, socklen);
 	}
 
-dbgprintf("CmpHost returns %d\n", ret);
 finalize_it:
 	return ret;
 }
 
 
+
+/* check if restrictions (ALCs) exists. The goal of this function is to disable the
+ * somewhat time-consuming ACL checks if no restrictions are defined (the usual case).
+ * This also permits to gain some speedup by using firewall-based ACLs instead of
+ * rsyslog ACLs (the recommended method.
+ * rgerhards, 2009-11-16
+ */
+static rsRetVal
+HasRestrictions(uchar *pszType, int *bHasRestrictions) {
+	struct AllowedSenders *pAllowRoot;
+	DEFiRet;
+
+	CHKiRet(setAllowRoot(&pAllowRoot, pszType));
+
+	*bHasRestrictions = (pAllowRoot == NULL) ? 0 : 1;
+
+finalize_it:
+	if(iRet != RS_RET_OK) {
+		*bHasRestrictions = 1; /* in this case it is better to check individually */
+		DBGPRINTF("Error %d trying to obtain ACL restriction state of '%s'\n", iRet, pszType);
+	}
+	RETiRet;
+}
+
+
 /* queryInterface function
  * rgerhards, 2008-03-05
  */
@@ -1562,12 +1603,14 @@ CODESTARTobjQueryInterface(net)
 	pIf->create_udp_socket = create_udp_socket;
 	pIf->closeUDPListenSockets = closeUDPListenSockets;
 	pIf->isAllowedSender = isAllowedSender;
+	pIf->isAllowedSender2 = isAllowedSender2;
 	pIf->should_use_so_bsdcompat = should_use_so_bsdcompat;
 	pIf->getLocalHostname = getLocalHostname;
 	pIf->AddPermittedPeer = AddPermittedPeer;
 	pIf->DestructPermittedPeers = DestructPermittedPeers;
 	pIf->PermittedPeerWildcardMatch = PermittedPeerWildcardMatch;
 	pIf->CmpHost = CmpHost;
+	pIf->HasRestrictions = HasRestrictions;
 finalize_it:
 ENDobjQueryInterface(net)
 
diff --git a/runtime/net.h b/runtime/net.h
index a50b6fcb..9e6fb9da 100644
--- a/runtime/net.h
+++ b/runtime/net.h
@@ -139,7 +139,7 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */
 	void (*debugListenInfo)(int fd, char *type);
 	int *(*create_udp_socket)(uchar *hostname, uchar *LogPort, int bIsServer);
 	void (*closeUDPListenSockets)(int *finet);
-	int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost);
+	int __attribute__((deprecated)) (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost);
 	rsRetVal (*getLocalHostname)(uchar**);
 	int (*should_use_so_bsdcompat)(void);
 	/* permitted peer handling should be replaced by something better (see comments above) */
@@ -148,11 +148,14 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */
 	rsRetVal (*PermittedPeerWildcardMatch)(permittedPeers_t *pPeer, uchar *pszNameToMatch, int *pbIsMatching);
 	/* v5 interface additions */
 	int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t);
+	/* v6 interface additions - 2009-11-16 */
+	rsRetVal (*HasRestrictions)(uchar *, int *bHasRestrictions);
+	int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS);
 	/* data members - these should go away over time... TODO */
 	int    pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */
 	int    pACLDontResolve;       /* add hostname to acl instead of resolving it to IP(s) */
 ENDinterface(net)
-#define netCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */
+#define netCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */
 
 /* prototypes */
 PROTOTYPEObj(net);
diff --git a/runtime/parser.c b/runtime/parser.c
index 38f72986..0686bd91 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -462,7 +462,8 @@ ParseMsg(msg_t *pMsg)
 	 * (and that functionality is too important for debugging to drop it...).
 	 */
 	DBGPRINTF("msg parser: flags %x, from '%s', msg '%.50s'\n", pMsg->msgFlags,
-		  getRcvFrom(pMsg), pMsg->pszRawMsg);
+		  (pMsg->msgFlags & NEEDS_DNSRESOL) ? UCHAR_CONSTANT("~NOTRESOLVED~") : getRcvFrom(pMsg),
+		  pMsg->pszRawMsg);
 
 	/* we now need to go through our list of parsers and see which one is capable of
 	 * parsing the message. Note that the first parser that requires message sanitization
diff --git a/runtime/wtp.c b/runtime/wtp.c
index 060e6627..ab7ca4bb 100644
--- a/runtime/wtp.c
+++ b/runtime/wtp.c
@@ -442,7 +442,6 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr)
 			CHKiRet(wtpStartWrkr(pThis));
 		}
 	} else {
-dbgprintf("YYY: wtpAdviseMaxWorkers, sufficient workers, just doing adivse signal cond busy\n");
 		pthread_cond_signal(pThis->pcondBusy);
 	}