diff options
| -rw-r--r-- | ChangeLog | 3 | ||||
| -rw-r--r-- | dirty.h | 2 | ||||
| -rw-r--r-- | doc/rsyslog_conf.html | 28 | ||||
| -rw-r--r-- | plugins/imgssapi/imgssapi.c | 12 | ||||
| -rw-r--r-- | plugins/imklog/bsd.c | 37 | ||||
| -rw-r--r-- | plugins/imtemplate/imtemplate.c | 2 | ||||
| -rw-r--r-- | plugins/imudp/imudp.c | 7 | ||||
| -rw-r--r-- | plugins/imuxsock/imuxsock.c | 27 | ||||
| -rw-r--r-- | plugins/omgssapi/omgssapi.c | 13 | ||||
| -rw-r--r-- | plugins/omrelp/omrelp.c | 4 | ||||
| -rw-r--r-- | runtime/conf.c | 13 | ||||
| -rw-r--r-- | runtime/glbl.c | 3 | ||||
| -rw-r--r-- | runtime/glbl.h | 1 | ||||
| -rw-r--r-- | tcps_sess.c | 41 | ||||
| -rw-r--r-- | tcps_sess.h | 2 | ||||
| -rw-r--r-- | tcpsrv.c | 2 | ||||
| -rw-r--r-- | tools/omfwd.c | 13 | ||||
| -rw-r--r-- | tools/syslogd.c | 79 |
18 files changed, 195 insertions, 94 deletions
@@ -1,5 +1,8 @@ --------------------------------------------------------------------------- Version 3.21.4 [DEVEL] (rgerhards), 2008-0???? +- removed compile time fixed message size limit (was 2K), limit can now + be set via $MaxMessageSize global config directive (finally gotten rid + of MAXLINE ;)) - enhanced doc for $ActionExecOnlyEveryNthTimeTimeout --------------------------------------------------------------------------- Version 3.21.3 [DEVEL] (rgerhards), 2008-08-13 @@ -27,8 +27,6 @@ #ifndef DIRTY_H_INCLUDED #define DIRTY_H_INCLUDED 1 -#define MAXLINE 2048 /* maximum line length */ - /* Flags to logmsg(). */ #define NOFLAG 0x000 /* no flag is set (to be used when a flag must be specified and none is required) */ diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 69e16ac6..0db69451 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -208,10 +208,30 @@ default 60000 (1 minute)]</li> </li> <li>$MainMsgQueueWorkerThreads <number>, num worker threads, default 1, recommended 1</li> -<li>$MainMsgQueueWorkerThreadMinumumMessages -<number>, default 100</li> -<li><a href="rsconf1_markmessageperiod.html">$MarkMessagePeriod</a> -(immark)</li> +<li>$MainMsgQueueWorkerThreadMinumumMessages <number>, default 100</li> +<li><a href="rsconf1_markmessageperiod.html">$MarkMessagePeriod</a> (immark)</li> +<li><b><i>$MaxMessageSize</i></b> <size_nbr>, default 2k - allows to specify maximum supported message size +(both for sending and receiving). The default +should be sufficient for almost all cases. Do not set this below 1k, as it would cause +interoperability problems with other syslog implementations.<br> +Change the setting to e.g. 32768 if you would like to +support large message sizes for IHE (32k is the current maximum +needed for IHE). I was initially tempted to set the default to 32k, +but there is a some memory footprint with the current +implementation in rsyslog. +<br>If you intend to receive Windows Event Log data (e.g. via +<a href="http://www.eventreporter.com/">EventReporter</a>), you might want to +increase this number to an even higher value, as event +log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea). +Note: testing showed that 4k seems to be +the typical maximum for <b>UDP</b> based syslog. This is an IP stack +restriction. Not always ... but very often. If you go beyond +that value, be sure to test that rsyslogd actually does what +you think it should do ;) It is highly suggested to use a TCP based transport +instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions. +<br>Note that 2k, the current default, is the smallest size that must be +supported in order to be compliant to the upcoming new syslog RFC series. +</li> <li><a href="rsconf1_moddir.html">$ModDir</a></li> <li><a href="rsconf1_modload.html">$ModLoad</a></li> <li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li> diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index 766cb519..cce6c40f 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -55,6 +55,7 @@ #include "tcps_sess.h" #include "errmsg.h" #include "netstrm.h" +#include "glbl.h" MODULE_TYPE_INPUT @@ -80,6 +81,7 @@ DEFobjCurrIf(gssutil) DEFobjCurrIf(errmsg) DEFobjCurrIf(netstrm) DEFobjCurrIf(net) +DEFobjCurrIf(glbl) static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ static gss_cred_id_t gss_server_creds = GSS_C_NO_CREDENTIAL; @@ -392,10 +394,14 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) allowedMethods = pGSrv->allowedMethods; if(allowedMethods & ALLOWEDMETHOD_GSS) { /* Buffer to store raw message in case that - * gss authentication fails halfway through. + * gss authentication fails halfway through. This buffer + * is currently dynamically allocated, for performance + * reasons we should look for a better way to do it. + * rgerhars, 2008-09-02 */ - char buf[MAXLINE]; + char *buf; int ret = 0; + CHKmalloc(buf = (char*) malloc(sizeof(char) * (glbl.GetMaxLine() + 1))); dbgprintf("GSS-API Trying to accept TCP session %p\n", pSess); @@ -648,6 +654,7 @@ CODESTARTmodExit objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(gssutil, LM_GSSUTIL_FILENAME); objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); objRelease(netstrm, LM_NETSTRM_FILENAME); objRelease(net, LM_NET_FILENAME); ENDmodExit @@ -695,6 +702,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(gssutil, LM_GSSUTIL_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(netstrm, LM_NETSTRM_FILENAME)); CHKiRet(objUse(net, LM_NET_FILENAME)); diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index 39b644c0..0a581081 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -110,15 +110,32 @@ klogWillRun(void) static void readklog(void) { - char *p, *q, line[MAXLINE + 1]; + char *p, *q; int len, i; + int iMaxLine; + uchar bufRcv[4096+1]; + uchar *pRcv = NULL; /* receive buffer */ + + iMaxLine = glbl.GetMaxLine(); + + /* we optimize performance: if iMaxLine is below 4K (which it is in almost all + * cases, we use a fixed buffer on the stack. Only if it is higher, heap memory + * is used. We could use alloca() to achive a similar aspect, but there are so + * many issues with alloca() that I do not want to take that route. + * rgerhards, 2008-09-02 + */ + if((size_t) iMaxLine < sizeof(bufRcv) - 1) { + pRcv = bufRcv; + } else { + CHKmalloc(pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))); + } len = 0; for (;;) { - dbgprintf("----------imklog waiting for kernel log line\n"); - i = read(fklog, line + len, MAXLINE - 1 - len); + dbgprintf("----------imklog(BSD) waiting for kernel log line\n"); + i = read(fklog, pRcv + len, iMaxLine - len); if (i > 0) { - line[i + len] = '\0'; + pRcv[i + len] = '\0'; } else { if (i < 0 && errno != EINTR && errno != EAGAIN) { imklogLogIntMsg(LOG_ERR, @@ -129,20 +146,24 @@ readklog(void) break; } - for (p = line; (q = strchr(p, '\n')) != NULL; p = q + 1) { + for (p = pRcv; (q = strchr(p, '\n')) != NULL; p = q + 1) { *q = '\0'; Syslog(LOG_INFO, (uchar*) p); } len = strlen(p); - if (len >= MAXLINE - 1) { + if (len >= iMaxLine - 1) { Syslog(LOG_INFO, (uchar*)p); len = 0; } if (len > 0) - memmove(line, p, len + 1); + memmove(pRcv, p, len + 1); } if (len > 0) - Syslog(LOG_INFO, (uchar*)line); + Syslog(LOG_INFO, pRcv); + +finalize_it: + if(pRcv != NULL && (size_t) iMaxLine >= sizeof(bufRcv) - 1) + free(pRcv); } diff --git a/plugins/imtemplate/imtemplate.c b/plugins/imtemplate/imtemplate.c index 6d29c4f1..c391d314 100644 --- a/plugins/imtemplate/imtemplate.c +++ b/plugins/imtemplate/imtemplate.c @@ -315,7 +315,7 @@ CODESTARTwillRun if(udpLstnSocks == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); - if((pRcvBuf = malloc(MAXLINE * sizeof(char))) == NULL) { + if((pRcvBuf = malloc(glbl.GetMaxLine * sizeof(char))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } * diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 6d3a075f..92d930d4 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -51,6 +51,7 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) +static int iMaxLine; /* maximum UDP message size supported */ static int *udpLstnSocks = NULL; /* Internet datagram sockets, first element is nbr of elements * read-only after init(), but beware of restart! */ static uchar *pszBindAddr = NULL; /* IP to bind socket to */ @@ -180,7 +181,7 @@ CODESTARTrunInput for (i = 0; nfds && i < *udpLstnSocks; i++) { if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { socklen = sizeof(frominet); - l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, MAXLINE - 1, 0, + l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if (l > 0) { if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) { @@ -231,7 +232,9 @@ CODESTARTwillRun if(udpLstnSocks == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); - if((pRcvBuf = malloc(MAXLINE * sizeof(char))) == NULL) { + iMaxLine = glbl.GetMaxLine(); + + if((pRcvBuf = malloc((iMaxLine + 1) * sizeof(char))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } finalize_it: diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 4f1fcea4..6e0fa1d3 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -196,14 +196,31 @@ static rsRetVal readSocket(int fd, int iSock) { DEFiRet; int iRcvd; - uchar line[MAXLINE +1]; + int iMaxLine; + uchar bufRcv[4096+1]; + uchar *pRcv = NULL; /* receive buffer */ assert(iSock >= 0); - iRcvd = recv(fd, line, MAXLINE - 1, 0); + + iMaxLine = glbl.GetMaxLine(); + + /* we optimize performance: if iMaxLine is below 4K (which it is in almost all + * cases, we use a fixed buffer on the stack. Only if it is higher, heap memory + * is used. We could use alloca() to achive a similar aspect, but there are so + * many issues with alloca() that I do not want to take that route. + * rgerhards, 2008-09-02 + */ + if((size_t) iMaxLine < sizeof(bufRcv) - 1) { + pRcv = bufRcv; + } else { + CHKmalloc(pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))); + } + + iRcvd = recv(fd, pRcv, iMaxLine, 0); dbgprintf("Message from UNIX socket: #%d\n", fd); if (iRcvd > 0) { parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], - (uchar*)"127.0.0.1", line, + (uchar*)"127.0.0.1", pRcv, iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock]); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; @@ -212,6 +229,10 @@ static rsRetVal readSocket(int fd, int iSock) errmsg.LogError(errno, NO_ERRCODE, "recvfrom UNIX"); } +finalize_it: + if(pRcv != NULL && (size_t) iMaxLine >= sizeof(bufRcv) - 1) + free(pRcv); + RETiRet; } diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 82fca2db..e0cc8af6 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -378,6 +378,7 @@ ENDtryResume BEGINdoAction char *psz; /* temporary buffering */ register unsigned l; + int iMaxLine; CODESTARTdoAction switch (pData->eDestState) { case eDestFORW_SUSP: @@ -392,10 +393,11 @@ CODESTARTdoAction case eDestFORW: dbgprintf(" %s:%s/%s\n", pData->f_hname, getFwdSyslogPt(pData), "tcp-gssapi"); + iMaxLine = glbl.GetMaxLine(); psz = (char*) ppString[0]; l = strlen((char*) psz); - if (l > MAXLINE) - l = MAXLINE; + if((int) l > iMaxLine) + l = iMaxLine; # ifdef USE_NETZIP /* Check if we should compress and, if so, do it. We also @@ -407,10 +409,14 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ if(pData->compressionLevel && (l > MIN_SIZE_FOR_COMPRESS)) { - Bytef out[MAXLINE+MAXLINE/100+12] = "z"; + Bytef *out; uLongf destLen = sizeof(out) / sizeof(Bytef); uLong srcLen = l; int ret; + /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ + CHKmalloc(out = (Bytef*) malloc(iMaxLine + iMaxLine/100 + 12)); + out[0] = 'z'; + out[1] = '\0'; ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, srcLen, pData->compressionLevel); dbgprintf("Compressing message, length was %d now %d, return state %d.\n", @@ -442,6 +448,7 @@ CODESTARTdoAction } break; } +finalize_it: ENDdoAction diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 71d6e797..8d74c82f 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -159,8 +159,8 @@ CODESTARTdoAction lenMsg = strlen((char*) pMsg); /* TODO: don't we get this? */ /* TODO: think about handling oversize messages! */ - if(lenMsg > MAXLINE) - lenMsg = MAXLINE; + if((int) lenMsg > glbl.GetMaxLine()) + lenMsg = glbl.GetMaxLine(); /* forward */ ret = relpCltSendSyslog(pData->pRelpClt, (uchar*) pMsg, lenMsg); diff --git a/runtime/conf.c b/runtime/conf.c index ffe67dbe..e55b8d18 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -570,8 +570,7 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int } -/* - * Helper to cfline(). This function takes the filter part of a traditional, PRI +/* Helper to cfline(). This function takes the filter part of a traditional, PRI * based line and decodes the PRIs given in the selector line. It processed the * line up to the beginning of the action part. A pointer to that beginnig is * passed back to the caller. @@ -587,8 +586,9 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f int pri; int singlpri = 0; int ignorepri = 0; - uchar buf[MAXLINE]; + uchar buf[2048]; /* buffer for facility and priority names */ uchar xbuf[200]; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -613,7 +613,7 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f continue; /* collect priority name */ - for (bp = buf; *q && !strchr("\t ,;", *q); ) + for (bp = buf; *q && !strchr("\t ,;", *q) && bp < buf+sizeof(buf)-1 ; ) *bp++ = *q++; *bp = '\0'; @@ -624,6 +624,7 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f /* decode priority name */ if ( *buf == '!' ) { ignorepri = 1; + /* copy below is ok, we can NOT go off the allocated area */ for (bp=buf; *(bp+1); bp++) *bp=*(bp+1); *bp='\0'; @@ -649,7 +650,7 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f /* scan facilities */ while (*p && !strchr("\t .;", *p)) { - for (bp = buf; *p && !strchr("\t ,;.", *p); ) + for (bp = buf; *p && !strchr("\t ,;.", *p) && bp < buf+sizeof(buf)-1 ; ) *bp++ = *p++; *bp = '\0'; if (*buf == '*') { @@ -732,7 +733,7 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f p++; *pline = p; - return RS_RET_OK; + RETiRet; } diff --git a/runtime/glbl.c b/runtime/glbl.c index 11a664f8..1114fcd3 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -51,6 +51,7 @@ DEFobjStaticHelpers * class... */ static uchar *pszWorkDir = NULL; +static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ static int option_DisallowWarning = 1; /* complain if message from disallowed sender is received */ @@ -84,6 +85,7 @@ static dataType Get##nameFunc(void) \ return(nameVar); \ } +SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, option_DisallowWarning, int) @@ -170,6 +172,7 @@ CODESTARTobjQueryInterface(glbl) #define SIMP_PROP(name) \ pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; + SIMP_PROP(MaxLine); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); diff --git a/runtime/glbl.h b/runtime/glbl.h index 90436319..0c83bdd5 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -40,6 +40,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ #define SIMP_PROP(name, dataType) \ dataType (*Get##name)(void); \ rsRetVal (*Set##name)(dataType); + SIMP_PROP(MaxLine, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) diff --git a/tcps_sess.c b/tcps_sess.c index b93bb115..0cb23fd0 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -46,19 +46,24 @@ /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) DEFobjCurrIf(netstrm) +static int iMaxLine; /* maximum size of a single message */ + /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); -/* Standard-Constructor - */ +/* Standard-Constructor */ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END macro! */ pThis->iMsg = 0; /* just make sure... */ pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ + /* now allocate the message reception buffer */ + CHKmalloc(pThis->pMsg = (uchar*) malloc(sizeof(uchar) * iMaxLine + 1)); +finalize_it: ENDobjConstruct(tcps_sess) @@ -92,6 +97,8 @@ CODESTARTobjDestruct(tcps_sess) free(pThis->fromHost); if(pThis->fromHostIP != NULL) free(pThis->fromHostIP); + if(pThis->pMsg != NULL) + free(pThis->pMsg); ENDobjDestruct(tcps_sess) @@ -222,7 +229,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->msg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); + parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); pThis->bAtStrtOfFram = 1; } @@ -288,23 +295,23 @@ processDataRcvd(tcps_sess_t *pThis, char c) dbgprintf("Framing Error: invalid octet count\n"); errmsg.LogError(0, NO_ERRCODE, "Framing Error in received TCP message: " "invalid octet count %d.\n", pThis->iOctetsRemain); - } else if(pThis->iOctetsRemain > MAXLINE) { + } else if(pThis->iOctetsRemain > iMaxLine) { /* while we can not do anything against it, we can at least log an indication * that something went wrong) -- rgerhards, 2008-03-14 */ - dbgprintf("truncating message with %d octets - MAXLINE is %d\n", - pThis->iOctetsRemain, MAXLINE); + dbgprintf("truncating message with %d octets - max msg size is %d\n", + pThis->iOctetsRemain, iMaxLine); errmsg.LogError(0, NO_ERRCODE, "received oversize message: size is %d bytes, " - "MAXLINE is %d, truncating...\n", pThis->iOctetsRemain, MAXLINE); + "max msg size is %d, truncating...\n", pThis->iOctetsRemain, iMaxLine); } pThis->inputState = eInMsg; } } else { assert(pThis->inputState == eInMsg); - if(pThis->iMsg >= MAXLINE) { + if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ - dbgprintf("error: message received is larger than MAXLINE, we split it\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->msg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); + dbgprintf("error: message received is larger than max msg size, we split it\n"); + parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -314,16 +321,16 @@ processDataRcvd(tcps_sess_t *pThis, char c) } if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->msg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); + parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! - * If we have a message that is larger than MAXLINE, we truncate it. This is the best + * If we have a message that is larger than the max msg size, we truncate it. This is the best * we can do in light of what the engine supports. -- rgerhards, 2008-03-14 */ - if(pThis->iMsg < MAXLINE) { - *(pThis->msg + pThis->iMsg++) = c; + if(pThis->iMsg < iMaxLine) { + *(pThis->pMsg + pThis->iMsg++) = c; } } @@ -332,7 +339,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->msg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); + parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY); pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } @@ -430,6 +437,10 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ + objRelease(glbl, CORE_COMPONENT); + /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, tcps_sessDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, tcps_sessConstructFinalize); diff --git a/tcps_sess.h b/tcps_sess.h index ff7c167a..576466ff 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -42,7 +42,7 @@ typedef struct tcps_sess_s { } inputState; /* our current state */ int iOctetsRemain; /* Number of Octets remaining in message */ TCPFRAMINGMODE eFraming; - uchar msg[MAXLINE+1]; + uchar *pMsg; /* message (fragment) received */ uchar *fromHost; uchar *fromHostIP; void *pUsr; /* a user-pointer */ @@ -450,7 +450,7 @@ Run(tcpsrv_t *pThis) while(nfds && iTCPSess != -1) { CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[MAXLINE]; + char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); /* Receive message */ diff --git a/tools/omfwd.c b/tools/omfwd.c index df2f0342..1dd184ef 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -386,16 +386,19 @@ ENDtryResume BEGINdoAction char *psz; /* temporary buffering */ register unsigned l; + int iMaxLine; CODESTARTdoAction CHKiRet(doTryResume(pData)); + iMaxLine = glbl.GetMaxLine(); + dbgprintf(" %s:%s/%s\n", pData->f_hname, getFwdPt(pData), pData->protocol == FORW_UDP ? "udp" : "tcp"); psz = (char*) ppString[0]; l = strlen((char*) psz); - if (l > MAXLINE) - l = MAXLINE; + if((int) l > iMaxLine) + l = iMaxLine; # ifdef USE_NETZIP /* Check if we should compress and, if so, do it. We also @@ -407,10 +410,14 @@ CODESTARTdoAction * rgerhards, 2006-11-30 */ if(pData->compressionLevel && (l > MIN_SIZE_FOR_COMPRESS)) { - Bytef out[MAXLINE+MAXLINE/100+12] = "z"; + Bytef *out; uLongf destLen = sizeof(out) / sizeof(Bytef); uLong srcLen = l; int ret; + /* TODO: optimize malloc sequence? -- rgerhards, 2008-09-02 */ + CHKmalloc(out = (Bytef*) malloc(iMaxLine + iMaxLine/100 + 12)); + out[0] = 'z'; + out[1] = '\0'; ret = compress2((Bytef*) out+1, &destLen, (Bytef*) psz, srcLen, pData->compressionLevel); dbgprintf("Compressing message, length was %d now %d, return state %d.\n", diff --git a/tools/syslogd.c b/tools/syslogd.c index 758d5ed1..bf42a7ef 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -58,34 +58,6 @@ #include "config.h" #include "rsyslog.h" -/* change the following setting to e.g. 32768 if you would like to - * support large message sizes for IHE (32k is the current maximum - * needed for IHE). I was initially tempted to increase it to 32k, - * but there is a large memory footprint with the current - * implementation in rsyslog. This will change as the processing - * changes, but I have re-set it to 1k, because the vast majority - * of messages is below that and the memory savings is huge, at - * least compared to the overall memory footprint. - * - * If you intend to receive Windows Event Log data (e.g. via - * EventReporter - www.eventreporter.com), you might want to - * increase this number to an even higher value, as event - * log messages can be very lengthy. - * rgerhards, 2005-07-05 - * - * during my recent testing, it showed that 4k seems to be - * the typical maximum for UDP based syslog. This is a IP stack - * restriction. Not always ... but very often. If you go beyond - * that value, be sure to test that rsyslogd actually does what - * you think it should do ;) Also, it is a good idea to check the - * doc set for anything on IHE - it most probably has information on - * message sizes. - * rgerhards, 2005-08-05 - * - * I have increased the default message size to 2048 to be in sync - * with recent IETF syslog standardization efforts. - * rgerhards, 2006-11-30 - */ #define DEFUPRI (LOG_USER|LOG_NOTICE) #define TIMERINTVL 30 /* interval for checking flush, mark */ @@ -708,9 +680,10 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa uchar *pMsg; uchar *pData; uchar *pEnd; - uchar tmpline[MAXLINE + 1]; + int iMaxLine; + uchar *tmpline = NULL; # ifdef USE_NETZIP - uchar deflateBuf[MAXLINE + 1]; + uchar *deflateBuf = NULL; uLongf iLenDefBuf; # endif @@ -719,6 +692,18 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa assert(msg != NULL); assert(len >= 0); + /* we first allocate work buffers large enough to hold the configured maximum + * size of a message. Over time, we should change this to a more optimal way, i.e. + * by calling the function with the actual length of the message to be parsed. + * rgerhards, 2008-09-02 + * + * TODO: optimize buffer handling */ + iMaxLine = glbl.GetMaxLine(); + CHKmalloc(tmpline = malloc(sizeof(uchar) * (iMaxLine + 1))); +# ifdef USE_NETZIP + CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); +# endif + /* we first check if we have a NUL character at the very end of the * message. This seems to be a frequent problem with a number of senders. * So I have now decided to drop these NULs. However, if they are intentional, @@ -755,14 +740,14 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ if(len > 0 && *msg == 'z') { /* compressed data present? (do NOT change order if conditions!) */ /* we have compressed data, so let's deflate it. We support a maximum - * message size of MAXLINE. If it is larger, an error message is logged + * message size of iMaxLine. If it is larger, an error message is logged * and the message is dropped. We do NOT try to decompress larger messages * as such might be used for denial of service. It might happen to later * builds that such functionality be added as an optional, operator-configurable * feature. */ int ret; - iLenDefBuf = MAXLINE; + iLenDefBuf = iMaxLine; ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, len-1); @@ -795,11 +780,11 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa # endif /* ifdef USE_NETZIP */ while(pData < pEnd) { - if(iMsg >= MAXLINE) { + if(iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if * we are at end of message or not... */ - if(iMsg == MAXLINE) { + if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType); } else { @@ -810,7 +795,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa * (I couldn't do any more smart things anyway...). * rgerhards, 2007-9-20 */ - dbgprintf("internal error: iMsg > MAXLINE in printchopped()\n"); + dbgprintf("internal error: iMsg > max msg size in printchopped()\n"); } FINALIZE; /* in this case, we are done... nothing left we can do */ } @@ -818,7 +803,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa /* changed to the sequence (somewhat) proposed in * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 */ - if(iMsg + 3 < MAXLINE) { /* do we have space? */ + if(iMsg + 3 < iMaxLine) { /* do we have space? */ *(pMsg + iMsg++) = cCCEscapeChar; *(pMsg + iMsg++) = '0'; *(pMsg + iMsg++) = '0'; @@ -838,7 +823,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa * we known now what's going on. * rgerhards, 2007-07-17 */ - if(iMsg + 3 < MAXLINE) { /* do we have space? */ + if(iMsg + 3 < iMaxLine) { /* do we have space? */ *(pMsg + iMsg++) = cCCEscapeChar; *(pMsg + iMsg++) = '0' + ((*pData & 0300) >> 6); *(pMsg + iMsg++) = '0' + ((*pData & 0070) >> 3); @@ -856,6 +841,12 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType); finalize_it: + if(tmpline != NULL) + free(tmpline); +# ifdef USE_NETZIP + if(deflateBuf != NULL) + free(deflateBuf); +# endif RETiRet; } @@ -2001,16 +1992,21 @@ static void doexit() } -/* set the action resume interval - */ +/* set the maximum message size */ +static rsRetVal setMaxMsgSize(void __attribute__((unused)) *pVal, int iNewVal) +{ + return glbl.SetMaxLine(iNewVal); +} + + +/* set the action resume interval */ static rsRetVal setActionResumeInterval(void __attribute__((unused)) *pVal, int iNewVal) { return actionSetGlobalResumeInterval(iNewVal); } -/* set the processes umask (upon configuration request) - */ +/* set the processes umask (upon configuration request) */ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { umask(iUmask); @@ -2726,6 +2722,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"moddir", 0, eCmdHdlrGetWord, NULL, &pModDir, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"errormessagestostderr", 0, eCmdHdlrBinary, NULL, &bErrMsgToStderr, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, setMaxMsgSize, NULL, NULL)); /* now add other modules handlers (we should work on that to be able to do it in ClassInit(), but so far * that is not possible). -- rgerhards, 2008-01-28 |