added option to use unlimited-size select() calls

Thanks to varmjofekoj for the patch

Signed-off-by: Rainer Gerhards <rgerhards@adiscon.com>

12 files changed, 141 insertions, 31 deletions

diff --git a/ChangeLog b/ChangeLog
index 3b16824d..e8273528 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,5 @@
+- added option to use unlimited-size select() calls
+  Thanks to varmjofekoj for the patch
 ---------------------------------------------------------------------------
 Version 4.4.2  [v4-stable] (rgerhards), 2009-10-09
 - bugfix: invalid handling of zero-sized messages, could lead to mis-
diff --git a/configure.ac b/configure.ac
index 66a2d70d..932a5fc5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -341,6 +341,21 @@ AC_ARG_ENABLE([fsstnd],
   ])
 
 
+# support for unlimited select() syscall
+AC_ARG_ENABLE(unlimited_select,
+        [AS_HELP_STRING([--enable-unlimited-select],[Enable unlimited select() syscall @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_unlimited_select="yes" ;;
+          no) enable_unlimited_select="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-unlimited-select) ;;
+         esac],
+        [enable_unlimited_select="no"]
+)
+if test "$enable_unlimited_select" = "yes"; then
+        AC_DEFINE(USE_UNLIMITED_SELECT, 1, [If defined, the select() syscall won't be limited to a particular number of file descriptors.])
+fi
+
+
 # debug
 AC_ARG_ENABLE(debug,
         [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])],
@@ -843,6 +858,7 @@ echo "    Zlib compression support enabled:         $enable_zlib"
 echo "    rsyslog runtime will be built:            $enable_rsyslogrt"
 echo "    rsyslogd will be built:                   $enable_rsyslogd"
 echo "    custom module 1 will be built:            $enable_cust1"
+echo "    Unlimited select() support enabled:       $enable_unlimited_select"
 echo
 echo "---{ input plugins }---"
 echo "    Klog functionality enabled:               $enable_klog ($os_type)"
diff --git a/gss-misc.c b/gss-misc.c
index c9220595..2bfaf9c5 100644
--- a/gss-misc.c
+++ b/gss-misc.c
@@ -51,11 +51,14 @@
 #include "obj.h"
 #include "errmsg.h"
 #include "gss-misc.h"
+#include "glbl.h"
+#include "unlimited_select.h"
 
 MODULE_TYPE_LIB
 
 /* static data */
 DEFobjStaticHelpers
+DEFobjCurrIf(glbl)
 DEFobjCurrIf(errmsg)
 
 static void display_status_(char *m, OM_uint32 code, int type)
@@ -108,28 +111,38 @@ static int read_all(int fd, char *buf, unsigned int nbyte)
 {
     int     ret;
     char   *ptr;
-    fd_set  rfds;
     struct timeval tv;
+#ifdef USE_UNLIMITED_SELECT
+    fd_set  *pRfds = malloc(glbl.GetFdSetSize());
+#else
+    fd_set  rfds;
+    fd_set *pRfds = &rfds;
+#endif
 
     for (ptr = buf; nbyte; ptr += ret, nbyte -= ret) {
-	    FD_ZERO(&rfds);
-	    FD_SET(fd, &rfds);
+	    FD_ZERO(pRfds);
+	    FD_SET(fd, pRfds);
 	    tv.tv_sec = 1;
 	    tv.tv_usec = 0;
 
-	    if ((ret = select(FD_SETSIZE, &rfds, NULL, NULL, &tv)) <= 0
-		|| !FD_ISSET(fd, &rfds))
+	    if ((ret = select(FD_SETSIZE, pRfds, NULL, NULL, &tv)) <= 0
+		|| !FD_ISSET(fd, pRfds)) {
+                    freeFdSet(pRfds);
 		    return ret;
+            }
 	    ret = recv(fd, ptr, nbyte, 0);
 	    if (ret < 0) {
 		    if (errno == EINTR)
 			    continue;
+                    freeFdSet(pRfds);
 		    return (ret);
 	    } else if (ret == 0) {
+                    freeFdSet(pRfds);
 		    return (ptr - buf);
 	    }
     }
 
+    freeFdSet(pRfds);
     return (ptr - buf);
 }
 
diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c
index d8791880..1aad6622 100644
--- a/plugins/imgssapi/imgssapi.c
+++ b/plugins/imgssapi/imgssapi.c
@@ -56,6 +56,7 @@
 #include "errmsg.h"
 #include "netstrm.h"
 #include "glbl.h"
+#include "unlimited_select.h"
 
 
 MODULE_TYPE_INPUT
@@ -414,15 +415,20 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess)
 		CHKiRet(netstrm.GetSock(pSess->pStrm, &fdSess)); // TODO: method access!
 		if (allowedMethods & ALLOWEDMETHOD_TCP) {
 			int len;
-			fd_set  fds;
 			struct timeval tv;
+#ifdef USE_UNLIMITED_SELECT
+                        fd_set *pFds = malloc(glbl.GetFdSetSize());
+#else
+                        fd_set fds;
+                        fd_set *pFds = &fds;
+#endif
 		
 			do {
-				FD_ZERO(&fds);
-				FD_SET(fdSess, &fds);
+				FD_ZERO(pFds);
+				FD_SET(fdSess, pFds);
 				tv.tv_sec = 1;
 				tv.tv_usec = 0;
-				ret = select(fdSess + 1, &fds, NULL, NULL, &tv);
+				ret = select(fdSess + 1, pFds, NULL, NULL, &tv);
 			} while (ret < 0 && errno == EINTR);
 			if (ret < 0) {
 				errmsg.LogError(0, RS_RET_ERR, "TCP session %p will be closed, error ignored\n", pSess);
@@ -475,6 +481,8 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess)
 				pGSess->allowedMethods = ALLOWEDMETHOD_TCP;
 				ABORT_FINALIZE(RS_RET_OK); // TODO: define good error codes
 			}
+
+                        freeFdSet(pFds);
 		}
 
 		context = &pGSess->gss_context;
diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index 6f4a6384..b5c97f2c 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -44,6 +44,7 @@
 #include "parser.h"
 #include "datetime.h"
 #include "unicode-helper.h"
+#include "unlimited_select.h"
 
 MODULE_TYPE_INPUT
 
@@ -256,12 +257,18 @@ BEGINrunInput
 	int maxfds;
 	int nfds;
 	int i;
-	fd_set readfds;
 	struct sockaddr_storage frominetPrev;
 	int bIsPermitted;
 	uchar fromHost[NI_MAXHOST];
 	uchar fromHostIP[NI_MAXHOST];
 	uchar fromHostFQDN[NI_MAXHOST];
+#ifdef USE_UNLIMITED_SELECT
+        fd_set  *pReadfds = malloc(glbl.GetFdSetSize());
+#else
+        fd_set  readfds;
+        fd_set *pReadfds = &readfds;
+#endif
+
 CODESTARTrunInput
 	/* start "name caching" algo by making sure the previous system indicator
 	 * is invalidated.
@@ -280,30 +287,30 @@ CODESTARTrunInput
 		 * is given without -a, we do not need to listen at all..
 		 */
 	        maxfds = 0;
-	        FD_ZERO (&readfds);
+	        FD_ZERO (pReadfds);
 
 		/* Add the UDP listen sockets to the list of read descriptors. */
 		for (i = 0; i < *udpLstnSocks; i++) {
 			if (udpLstnSocks[i+1] != -1) {
 				if(Debug)
 					net.debugListenInfo(udpLstnSocks[i+1], "UDP");
-				FD_SET(udpLstnSocks[i+1], &readfds);
+				FD_SET(udpLstnSocks[i+1], pReadfds);
 				if(udpLstnSocks[i+1]>maxfds) maxfds=udpLstnSocks[i+1];
 			}
 		}
 		if(Debug) {
 			dbgprintf("--------imUDP calling select, active file descriptors (max %d): ", maxfds);
 			for (nfds = 0; nfds <= maxfds; ++nfds)
-				if ( FD_ISSET(nfds, &readfds) )
+				if ( FD_ISSET(nfds, pReadfds) )
 					dbgprintf("%d ", nfds);
 			dbgprintf("\n");
 		}
 
 		/* wait for io to become ready */
-		nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL);
+		nfds = select(maxfds+1, (fd_set *) pReadfds, NULL, NULL, NULL);
 
 	       for(i = 0; nfds && i < *udpLstnSocks; i++) {
-			if(FD_ISSET(udpLstnSocks[i+1], &readfds)) {
+			if(FD_ISSET(udpLstnSocks[i+1], pReadfds)) {
 		       		processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted,
 					      fromHost, fromHostFQDN, fromHostIP);
 			--nfds; /* indicate we have processed one descriptor */
@@ -312,6 +319,7 @@ CODESTARTrunInput
 	       /* end of a run, back to loop for next recv() */
 	}
 
+	freeFdSet(pReadfds);
 	return iRet;
 ENDrunInput
 
diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c
index 1d88a2b5..28f8d6b5 100644
--- a/plugins/imuxsock/imuxsock.c
+++ b/plugins/imuxsock/imuxsock.c
@@ -43,6 +43,7 @@
 #include "net.h"
 #include "glbl.h"
 #include "msg.h"
+#include "unlimited_select.h"
 
 MODULE_TYPE_INPUT
 
@@ -245,7 +246,13 @@ BEGINrunInput
 	int nfds;
 	int i;
 	int fd;
-	fd_set readfds;
+#ifdef USE_UNLIMITED_SELECT
+        fd_set  *pReadfds = malloc(glbl.GetFdSetSize());
+#else
+        fd_set  readfds;
+        fd_set *pReadfds = &readfds;
+#endif
+
 CODESTARTrunInput
 	/* this is an endless loop - it is terminated when the thread is
 	 * signalled to do so. This, however, is handled by the framework,
@@ -259,11 +266,11 @@ CODESTARTrunInput
 		 * is given without -a, we do not need to listen at all..
 		 */
 	        maxfds = 0;
-	        FD_ZERO (&readfds);
+	        FD_ZERO (pReadfds);
 		/* Copy master connections */
 		for (i = startIndexUxLocalSockets; i < nfunix; i++) {
 			if (funix[i] != -1) {
-				FD_SET(funix[i], &readfds);
+				FD_SET(funix[i], pReadfds);
 				if (funix[i]>maxfds) maxfds=funix[i];
 			}
 		}
@@ -271,22 +278,23 @@ CODESTARTrunInput
 		if(Debug) {
 			dbgprintf("--------imuxsock calling select, active file descriptors (max %d): ", maxfds);
 			for (nfds= 0; nfds <= maxfds; ++nfds)
-				if ( FD_ISSET(nfds, &readfds) )
+				if ( FD_ISSET(nfds, pReadfds) )
 					dbgprintf("%d ", nfds);
 			dbgprintf("\n");
 		}
 
 		/* wait for io to become ready */
-		nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL);
+		nfds = select(maxfds+1, (fd_set *) pReadfds, NULL, NULL, NULL);
 
 		for (i = 0; i < nfunix && nfds > 0; i++) {
-			if ((fd = funix[i]) != -1 && FD_ISSET(fd, &readfds)) {
+			if ((fd = funix[i]) != -1 && FD_ISSET(fd, pReadfds)) {
 				readSocket(fd, i);
 				--nfds; /* indicate we have processed one */
 			}
 		}
 	}
 
+	freeFdSet(pReadfds);
 	RETiRet;
 ENDrunInput
 
diff --git a/runtime/Makefile.am b/runtime/Makefile.am
index eeb656d6..006b7459 100644
--- a/runtime/Makefile.am
+++ b/runtime/Makefile.am
@@ -15,6 +15,7 @@ librsyslog_la_SOURCES = \
 	nsd.h \
 	glbl.h \
 	glbl.c \
+	unlimited_select.h \
 	conf.c \
 	conf.h \
 	parser.h \
diff --git a/runtime/glbl.c b/runtime/glbl.c
index 28f14320..a443b948 100644
--- a/runtime/glbl.c
+++ b/runtime/glbl.c
@@ -68,6 +68,9 @@ static uchar *pszDfltNetstrmDrvr = NULL; /* module name of default netstream dri
 static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm driver */
 static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */
 static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */
+#ifdef USE_UNLIMITED_SELECT
+static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */
+#endif
 
 
 /* define a macro for the simple properties' set and get functions
@@ -100,6 +103,9 @@ SIMP_PROP(DisableDNS, bDisableDNS, int)
 SIMP_PROP(LocalDomain, LocalDomain, uchar*)
 SIMP_PROP(StripDomains, StripDomains, char**)
 SIMP_PROP(LocalHosts, LocalHosts, char**)
+#ifdef USE_UNLIMITED_SELECT
+SIMP_PROP(FdSetSize, iFdSetSize, int)
+#endif
 
 SIMP_PROP_SET(LocalFQDNName, LocalFQDNName, uchar*)
 SIMP_PROP_SET(LocalHostName, LocalHostName, uchar*)
@@ -217,6 +223,9 @@ CODESTARTobjQueryInterface(glbl)
 	SIMP_PROP(DfltNetstrmDrvrCAF)
 	SIMP_PROP(DfltNetstrmDrvrKeyFile)
 	SIMP_PROP(DfltNetstrmDrvrCertFile)
+#ifdef USE_UNLIMITED_SELECT
+	SIMP_PROP(FdSetSize)
+#endif
 #undef	SIMP_PROP
 finalize_it:
 ENDobjQueryInterface(glbl)
@@ -251,6 +260,9 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	bOptimizeUniProc = 1;
 	bHUPisRestart = 1;
 	bPreserveFQDN = 0;
+#ifdef USE_UNLIMITED_SELECT
+	iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask);
+#endif
 	return RS_RET_OK;
 }
 
diff --git a/runtime/glbl.h b/runtime/glbl.h
index 5bdf4f57..fb72a965 100644
--- a/runtime/glbl.h
+++ b/runtime/glbl.h
@@ -57,6 +57,9 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */
 	SIMP_PROP(DfltNetstrmDrvrCAF, uchar*)
 	SIMP_PROP(DfltNetstrmDrvrKeyFile, uchar*)
 	SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*)
+#ifdef USE_UNLIMITED_SELECT
+	SIMP_PROP(FdSetSize, int)
+#endif
 #undef	SIMP_PROP
 ENDinterface(glbl)
 #define glblCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */
diff --git a/runtime/nsdsel_ptcp.c b/runtime/nsdsel_ptcp.c
index 41b85e0c..e2cfca7c 100644
--- a/runtime/nsdsel_ptcp.c
+++ b/runtime/nsdsel_ptcp.c
@@ -36,6 +36,7 @@
 #include "errmsg.h"
 #include "nsd_ptcp.h"
 #include "nsdsel_ptcp.h"
+#include "unlimited_select.h"
 
 /* static data */
 DEFobjStaticHelpers
@@ -47,14 +48,23 @@ DEFobjCurrIf(glbl)
  */
 BEGINobjConstruct(nsdsel_ptcp) /* be sure to specify the object type also in END macro! */
 	pThis->maxfds = 0;
+#ifdef USE_UNLIMITED_SELECT
+        pThis->pReadfds = calloc(1, glbl.GetFdSetSize());
+        pThis->pWritefds = calloc(1, glbl.GetFdSetSize());
+#else
 	FD_ZERO(&pThis->readfds);
 	FD_ZERO(&pThis->writefds);
+#endif
 ENDobjConstruct(nsdsel_ptcp)
 
 
 /* destructor for the nsdsel_ptcp object */
 BEGINobjDestruct(nsdsel_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */
 CODESTARTobjDestruct(nsdsel_ptcp)
+#ifdef USE_UNLIMITED_SELECT
+	freeFdSet(pThis->pReadfds);
+	freeFdSet(pThis->pWritefds);
+#endif
 ENDobjDestruct(nsdsel_ptcp)
 
 
@@ -65,20 +75,27 @@ Add(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp)
 	DEFiRet;
 	nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel;
 	nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd;
+#ifdef USE_UNLIMITED_SELECT
+        fd_set *pReadfds = pThis->pReadfds;
+        fd_set *pWritefds = pThis->pWritefds;
+#else
+        fd_set *pReadfds = &pThis->readfds;
+        fd_set *pWritefds = &pThis->writefds;
+#endif
 
 	ISOBJ_TYPE_assert(pSock, nsd_ptcp);
 	ISOBJ_TYPE_assert(pThis, nsdsel_ptcp);
 
 	switch(waitOp) {
 		case NSDSEL_RD:
-			FD_SET(pSock->sock, &pThis->readfds);
+			FD_SET(pSock->sock, pReadfds);
 			break;
 		case NSDSEL_WR:
-			FD_SET(pSock->sock, &pThis->writefds);
+			FD_SET(pSock->sock, pWritefds);
 			break;
 		case NSDSEL_RDWR:
-			FD_SET(pSock->sock, &pThis->readfds);
-			FD_SET(pSock->sock, &pThis->writefds);
+			FD_SET(pSock->sock, pReadfds);
+			FD_SET(pSock->sock, pWritefds);
 			break;
 	}
 
@@ -98,6 +115,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady)
 	DEFiRet;
 	int i;
 	nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel;
+#ifdef USE_UNLIMITED_SELECT
+        fd_set *pReadfds = pThis->pReadfds;
+        fd_set *pWritefds = pThis->pWritefds;
+#else
+        fd_set *pReadfds = &pThis->readfds;
+        fd_set *pWritefds = &pThis->writefds;
+#endif
 
 	ISOBJ_TYPE_assert(pThis, nsdsel_ptcp);
 	assert(piNumReady != NULL);
@@ -106,13 +130,13 @@ Select(nsdsel_t *pNsdsel, int *piNumReady)
 		// TODO: name in dbgprintf!
 		dbgprintf("--------<NSDSEL_PTCP> calling select, active fds (max %d): ", pThis->maxfds);
 		for(i = 0; i <= pThis->maxfds; ++i)
-			if(FD_ISSET(i, &pThis->readfds) || FD_ISSET(i, &pThis->writefds))
+			if(FD_ISSET(i, pReadfds) || FD_ISSET(i, pWritefds))
 				dbgprintf("%d ", i);
 		dbgprintf("\n");
 	}
 
 	/* now do the select */
-	*piNumReady = select(pThis->maxfds+1, &pThis->readfds, &pThis->writefds, NULL, NULL);
+	*piNumReady = select(pThis->maxfds+1, pReadfds, pWritefds, NULL, NULL);
 
 	RETiRet;
 }
@@ -125,6 +149,13 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady)
 	DEFiRet;
 	nsdsel_ptcp_t *pThis = (nsdsel_ptcp_t*) pNsdsel;
 	nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd;
+#ifdef USE_UNLIMITED_SELECT
+        fd_set *pReadfds = pThis->pReadfds;
+        fd_set *pWritefds = pThis->pWritefds;
+#else
+        fd_set *pReadfds = &pThis->readfds;
+        fd_set *pWritefds = &pThis->writefds;
+#endif
 
 	ISOBJ_TYPE_assert(pThis, nsdsel_ptcp);
 	ISOBJ_TYPE_assert(pSock, nsd_ptcp);
@@ -132,14 +163,14 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady)
 
 	switch(waitOp) {
 		case NSDSEL_RD:
-			*pbIsReady = FD_ISSET(pSock->sock, &pThis->readfds);
+			*pbIsReady = FD_ISSET(pSock->sock, pReadfds);
 			break;
 		case NSDSEL_WR:
-			*pbIsReady = FD_ISSET(pSock->sock, &pThis->writefds);
+			*pbIsReady = FD_ISSET(pSock->sock, pWritefds);
 			break;
 		case NSDSEL_RDWR:
-			*pbIsReady =   FD_ISSET(pSock->sock, &pThis->readfds)
-				     | FD_ISSET(pSock->sock, &pThis->writefds);
+			*pbIsReady =   FD_ISSET(pSock->sock, pReadfds)
+				     | FD_ISSET(pSock->sock, pWritefds);
 			break;
 	}
 
diff --git a/runtime/nsdsel_ptcp.h b/runtime/nsdsel_ptcp.h
index 6c0c7fa7..f9ec8210 100644
--- a/runtime/nsdsel_ptcp.h
+++ b/runtime/nsdsel_ptcp.h
@@ -31,8 +31,13 @@ typedef nsdsel_if_t nsdsel_ptcp_if_t; /* we just *implement* this interface */
 struct nsdsel_ptcp_s {
 	BEGINobjInstance;	/* Data to implement generic object - MUST be the first data element! */
 	int maxfds;
+#ifdef USE_UNLIMITED_SELECT
+	fd_set *pReadfds;
+	fd_set *pWritefds;
+#else
 	fd_set readfds;
 	fd_set writefds;
+#endif
 };
 
 /* interface is defined in nsd.h, we just implement it! */
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 5f6b480f..687f2217 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -2150,6 +2150,9 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles)
 				iFiles, errStr, (long) maxFiles.rlim_max);
 		ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE);
 	}
+#ifdef USE_UNLIMITED_SELECT
+	glbl.SetFdSetSize(howmany(iFiles, __NFDBITS) * sizeof (fd_mask));
+#endif
 	DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max);
 
 finalize_it: