From 53c1a7305e119b397c3e80acfdd4a6aecc7ce121 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 3 Jul 2007 12:39:13 +0000 Subject: added support for dynamic file names in selector lines. Can now be created with templates. --- NEWS | 4 + linux/Makefile | 2 +- stringbuf.c | 17 ++++- stringbuf.h | 11 ++- syslogd.c | 230 +++++++++++++++++++++++++++++++++++++++++++++++++-------- template.h | 4 +- 6 files changed, 232 insertions(+), 36 deletions(-) diff --git a/NEWS b/NEWS index 8112b03a..860ddfe2 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- Version 1.15.0 (RGer), 2007-07-?? +- added ability to dynamically generate file names based on templates + and thus properties. This was a much-requested feature. It makes + life easy when it eG comes to splitting files based on the sender + address. --------------------------------------------------------------------------- Version 1.14.2 (RGer), 2007-07-03 ** this release fixes all known nits with IPv6 ** diff --git a/linux/Makefile b/linux/Makefile index 5bd22e06..f4435725 100644 --- a/linux/Makefile +++ b/linux/Makefile @@ -47,7 +47,7 @@ FEATURE_PTHREADS=1 FEATURE_KLOGD=1 # Enable debug mode (much slower code) -FEATURE_DEBUG=0 +FEATURE_DEBUG=1 # The following defines tell us where liblogging is located. This # is only needed if we build with RFC 3195 support. By default, diff --git a/stringbuf.c b/stringbuf.c index 5aa94fc4..fa26dc09 100755 --- a/stringbuf.c +++ b/stringbuf.c @@ -118,14 +118,14 @@ void rsCStrDestruct(rsCStrObj *pThis) } -rsRetVal rsCStrAppendStr(rsCStrObj *pThis, char* psz) +rsRetVal rsCStrAppendStrWithLen(rsCStrObj *pThis, char* psz, size_t iStrLen) { rsRetVal iRet; int iOldAllocInc; - int iStrLen; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); assert(psz != NULL); + assert(iStrLen >= 0); /* we first check if the to-be-added string is larger than the * alloc increment. If so, we temporarily increase the alloc @@ -139,7 +139,7 @@ rsRetVal rsCStrAppendStr(rsCStrObj *pThis, char* psz) * overwrite it below, this is faster than any if-construct. */ iOldAllocInc = pThis->iAllocIncrement; - if((iStrLen = strlen(psz)) > pThis->iAllocIncrement) { + if(iStrLen > pThis->iAllocIncrement) { pThis->iAllocIncrement = iStrLen; } @@ -152,6 +152,17 @@ rsRetVal rsCStrAppendStr(rsCStrObj *pThis, char* psz) } +/* changed to be a wrapper to rsCStrAppendStrWithLen() so that + * we can save some time when we have the length but do not + * need to change existing code. + * rgerhards, 2007-07-03 + */ +rsRetVal rsCStrAppendStr(rsCStrObj *pThis, char* psz) +{ + return rsCStrAppendStrWithLen(pThis, psz, strlen(psz)); +} + + rsRetVal rsCStrAppendInt(rsCStrObj *pThis, int i) { rsRetVal iRet; diff --git a/stringbuf.h b/stringbuf.h index c8359ab8..5f9a0879 100755 --- a/stringbuf.h +++ b/stringbuf.h @@ -72,12 +72,21 @@ rsRetVal rsCStrTruncate(rsCStrObj *pThis, int nTrunc); rsRetVal rsCStrTrimTrailingWhiteSpace(rsCStrObj *pThis); /** - * Append a string to the buffer. + * Append a string to the buffer. For performance reasons, + * use rsCStrAppenStrWithLen() if you know the length. * * \param psz pointer to string to be appended. Must not be NULL. */ rsRetVal rsCStrAppendStr(rsCStrObj *pThis, char* psz); +/** + * Append a string to the buffer. + * + * \param psz pointer to string to be appended. Must not be NULL. + * \param iStrLen the length of the string pointed to by psz + */ +rsRetVal rsCStrAppendStrWithLen(rsCStrObj *pThis, char* psz, size_t iStrLen); + /** * Set a new allocation incremet. This will influence * the allocation the next time the string will be expanded. diff --git a/syslogd.c b/syslogd.c index 271d70b9..6ea5260e 100644 --- a/syslogd.c +++ b/syslogd.c @@ -656,7 +656,7 @@ struct filed { char f_hname[MAXHOSTNAMELEN+1]; struct addrinfo *f_addr; int compressionLevel; /* 0 - no compression, else level for zlib */ - char * port; + char *port; int protocol; TCPFRAMINGMODE tcp_framing; # define FORW_UDP 0 @@ -673,7 +673,11 @@ struct filed { pthread_mutex_t mtxTCPSend; # endif } f_forw; /* forwarding address */ - char f_fname[MAXFNAME]; + struct { + char f_fname[MAXFNAME]; + char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ + char *pCurrName; /* name currently open, if dynamic name */ + } f_file; } f_un; char f_lasttime[16]; /* time of last occurrence */ char f_prevhost[MAXHOSTNAMELEN+1]; /* host from which recd. */ @@ -4688,7 +4692,7 @@ static void processMsg(struct msg *pMsg) memset(&emergfile, 0, sizeof(emergfile)); f = &emergfile; emergfile.f_type = F_TTY; - (void) strcpy(emergfile.f_un.f_fname, ttyname(0)); + (void) strcpy(emergfile.f_un.f_file.f_fname, ttyname(0)); cflineSetTemplateAndIOV(&emergfile, " TradFmt"); f->f_file = open(ttyname(0), O_WRONLY|O_NOCTTY); @@ -5710,6 +5714,97 @@ void iovCreate(struct filed *f) return; } +/* This functions converts a template into a string. It should + * actually be in template.c, but this requires larger re-structuring + * of the code (because all the property-access functions are static + * to this module). I have placed it next to the iov*() functions, as + * it is somewhat similiar in what it does. + * + * The function takes a template name and a pointer to a msg object. + * It the creates a string based on the template definition. A pointer + * to that string is returned to the caller. The caller MUST FREE that + * pointer when it is no longer needed. If the function fails, NULL + * is returned. + * If memory allocation fails in this function, we silently return + * NULL. The reason is that we can not do anything against it. And + * if we raise an alert, the memory situation might become even + * worse. So we prefer to let the caller deal with it. + * rgerhards, 2007-07-03 + */ +static char *tplToString(char *tplName, struct msg *pMsg) +{ + struct template *pTpl; + struct templateEntry *pTpe; + rsCStrObj *pCStr; + unsigned short bMustBeFreed; + char *pVal; + size_t iLenVal; + rsRetVal iRet; + + assert(tplName != NULL); + assert(pMsg != NULL); + + if((pTpl = tplFind(tplName, strlen(tplName))) == NULL) + return NULL; + + /* we now loop through the template. We obtain one value + * and copy it over to our dynamic string buffer. Then, we + * free the obtained value (if requested). We continue this + * loop until we got hold of all values. + */ + if((pCStr = rsCStrConstruct()) == NULL) { + dprintf("memory shortage, tplToString failed\n"); + return NULL; + } + + pTpe = pTpl->pEntryRoot; + while(pTpe != NULL) { + if(pTpe->eEntryType == CONSTANT) { + if((iRet = rsCStrAppendStrWithLen(pCStr, + pTpe->data.constant.pConstant, + pTpe->data.constant.iLenConstant) + ) != RS_RET_OK) { + dprintf("error %d during tplToString()\n", iRet); + /* it does not make sense to continue now */ + rsCStrDestruct(pCStr); + return NULL; + } + } else if(pTpe->eEntryType == FIELD) { + pVal = MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); + iLenVal = strlen(pVal); + /* we now need to check if we should use SQL option. In this case, + * we must go over the generated string and escape '\'' characters. + * rgerhards, 2005-09-22: the option values below look somewhat misplaced, + * but they are handled in this way because of legacy (don't break any + * existing thing). + */ + if(pTpl->optFormatForSQL == 1) + doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); + else if(pTpl->optFormatForSQL == 2) + doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); + /* value extracted, so lets copy */ + if((iRet = rsCStrAppendStrWithLen(pCStr, pVal, iLenVal)) != RS_RET_OK) { + dprintf("error %d during tplToString()\n", iRet); + /* it does not make sense to continue now */ + rsCStrDestruct(pCStr); + if(bMustBeFreed) + free(pVal); + return NULL; + } + if(bMustBeFreed) + free(pVal); + } + pTpe = pTpe->pNext; + } + + /* we are done with the template, now let's convert the result into a + * "real" (usable) string and discard the helper structures. + */ + rsCStrFinish(pCStr); + return rsCStrConvSzStrAndDestruct(pCStr); +} + + /* rgerhards 2005-06-21: Try to resolve a size limit * situation. This first runs the command, and then * checks if we are still above the treshold. @@ -5731,7 +5826,7 @@ int resolveFileSizeLimit(struct filed *f) */ system(f->f_sizeLimitCmd); - f->f_file = open(f->f_un.f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + f->f_file = open(f->f_un.f_file.f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0644); actualFileSize = lseek(f->f_file, 0, SEEK_END); @@ -5743,6 +5838,42 @@ int resolveFileSizeLimit(struct filed *f) return 0; } + +/* This function handles dynamic file names. It generates a new one + * based on the current message, checks if that file is already open + * and, if not, does everything needed to switch to the new one. + * Function returns 0 if all went well and non-zero otherwise. + * This is a helper to writeFile(). rgerhards, 2007-07-03 + */ +static int prepareDynFile(struct filed *f) +{ + char *newFileName; + + assert(f != NULL); + if((newFileName = tplToString(f->f_un.f_file.f_fname, f->f_pMsg)) == NULL) { + /* memory shortage - there is nothing we can do to resolve it. + * We silently ignore it, this is probably the best we can do. + */ + dprintf("prepareDynfile(): could not create file name, discarding this reques\n"); + return -1; + } + if(strcmp(newFileName, f->f_un.f_file.pCurrName)) { + dprintf("Requested log file different from currently open one - switching.\n"); + dprintf("Current file: '%s'\n", f->f_un.f_file.pCurrName); + dprintf("New file : '%s'\n", newFileName); + close(f->f_file); + f->f_file = open(newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0644); + free(f->f_un.f_file.pCurrName); + f->f_un.f_file.pCurrName = newFileName; + + } else { /* we are all set, the log file is already open */ + free(newFileName); + } + + return 0; +} + + /* rgerhards 2004-11-11: write to a file output. This * will be called for all outputs using file semantics, * for example also for pipes. @@ -5750,12 +5881,20 @@ int resolveFileSizeLimit(struct filed *f) void writeFile(struct filed *f) { off_t actualFileSize; + assert(f != NULL); + /* first check if we have a dynamic file name and, if so, + * check if it still is ok or a new file needs to be created + */ + if(f->f_un.f_file.bDynamicName) + if(prepareDynFile(f) != 0) + return; + /* create the message based on format specified */ iovCreate(f); again: - /* first check if we have a file size limit and, if so, + /* check if we have a file size limit and, if so, * obey to it. */ if(f->f_sizeLimit != 0) { @@ -5773,14 +5912,14 @@ again: f->f_type = F_UNUSED; snprintf(errMsg, sizeof(errMsg), "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - f->f_un.f_fname, (long long) f->f_sizeLimit, (long long) actualFileSize); + f->f_un.f_file.f_fname, (long long) f->f_sizeLimit, (long long) actualFileSize); errno = 0; logerror(errMsg); return; } else { snprintf(errMsg, sizeof(errMsg), "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - f->f_un.f_fname, (long long) f->f_sizeLimit, (long long) actualFileSize); + f->f_un.f_file.f_fname, (long long) f->f_sizeLimit, (long long) actualFileSize); errno = 0; logerror(errMsg); } @@ -5806,10 +5945,10 @@ again: #else && e == EBADF) { #endif - f->f_file = open(f->f_un.f_fname, O_WRONLY|O_APPEND|O_NOCTTY); + f->f_file = open(f->f_un.f_file.f_fname, O_WRONLY|O_APPEND|O_NOCTTY); if (f->f_file < 0) { f->f_type = F_UNUSED; - logerror(f->f_un.f_fname); + logerror(f->f_un.f_file.f_fname); } else { untty(); goto again; @@ -5817,7 +5956,7 @@ again: } else { f->f_type = F_UNUSED; errno = e; - logerror(f->f_un.f_fname); + logerror(f->f_un.f_file.f_fname); } } else if (f->f_flags & SYNC_FILE) fsync(f->f_file); @@ -6050,7 +6189,7 @@ void fprintlog(register struct filed *f) case F_TTY: case F_FILE: case F_PIPE: - dprintf(" (%s)\n", f->f_un.f_fname); + dprintf(" (%s)\n", f->f_un.f_file.f_fname); /* TODO: check if we need f->f_time = now;*/ /* f->f_file == -1 is an indicator that the we couldn't open the file at startup. */ @@ -6080,13 +6219,13 @@ void fprintlog(register struct filed *f) l = f->f_iLenpsziov; if (l > MAXLINE) l = MAXLINE; - esize = strlen(f->f_un.f_fname) + strlen(psz) + 4; + esize = strlen(f->f_un.f_file.f_fname) + strlen(psz) + 4; if((pCSCmdLine = rsCStrConstruct()) == NULL) { /* nothing smart we can do - just keep going... */ dprintf("memory shortage - can not execute\n"); break; } - if((iRet = rsCStrAppendStr(pCSCmdLine, f->f_un.f_fname)) != RS_RET_OK) { + if((iRet = rsCStrAppendStr(pCSCmdLine, f->f_un.f_file.f_fname)) != RS_RET_OK) { dprintf("error %d during build command line(1)\n", iRet); break; } @@ -7070,13 +7209,15 @@ static void init() case F_PIPE: case F_TTY: case F_CONSOLE: - printf("%s", f->f_un.f_fname); + printf("%s", f->f_un.f_file.f_fname); if (f->f_file == -1) printf(" (unused)"); + if(f->f_un.f_file.bDynamicName) + printf(" (dynamic name)"); break; case F_SHELL: - printf("%s", f->f_un.f_fname); + printf("%s", f->f_un.f_file.f_fname); break; case F_FORW: @@ -7216,8 +7357,8 @@ static void cflineParseFileName(struct filed *f, char* p) f->f_type = F_FILE; } - pName = f->f_un.f_fname; - i = 1; /* we start at 1 so that we resever space for the '\0'! */ + pName = f->f_un.f_file.f_fname; + i = 1; /* we start at 1 so that we reseve space for the '\0'! */ while(*p && *p != ';' && i < MAXFNAME) { *pName++ = *p++; ++i; @@ -7240,7 +7381,7 @@ static void cflineParseFileName(struct filed *f, char* p) cflineSetTemplateAndIOV(f, szTemplateName); - dprintf("filename: '%s', template: '%s'\n", f->f_un.f_fname, szTemplateName); + dprintf("filename: '%s', template: '%s'\n", f->f_un.f_file.f_fname, szTemplateName); } @@ -7302,9 +7443,9 @@ static void cflineParseOutchannel(struct filed *f, char* p) } /* OK, we finally got a correct template. So let's use it... */ - strncpy(f->f_un.f_fname, pOch->pszFileTemplate, MAXFNAME); + strncpy(f->f_un.f_file.f_fname, pOch->pszFileTemplate, MAXFNAME); f->f_sizeLimit = pOch->uSizeLimit; - /* WARNING: It is dangerous "just" to pass the pointer. As wer + /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ f->f_sizeLimitCmd = pOch->cmdOnSizeLimit; @@ -7325,7 +7466,7 @@ static void cflineParseOutchannel(struct filed *f, char* p) cflineSetTemplateAndIOV(f, szBuf); - dprintf("[outchannel]filename: '%s', template: '%s', size: %lu\n", f->f_un.f_fname, szBuf, + dprintf("[outchannel]filename: '%s', template: '%s', size: %lu\n", f->f_un.f_file.f_fname, szBuf, f->f_sizeLimit); } @@ -7942,10 +8083,40 @@ static rsRetVal cfline(char *line, register struct filed *f) * traditional mode lines. */ cflineParseOutchannel(f, p); - f->f_file = open(f->f_un.f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + f->f_file = open(f->f_un.f_file.f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0644); break; + case '?': /* This is much like a regular file handle, but we need to obtain + * a template name. rgerhards, 2007-07-03 + */ + ++p; /* eat '?' */ + cflineParseFileName(f, p); + if(f->f_type == F_UNUSED) + /* safety measure to make sure we have a valid + * selector line before we continue down below. + * rgerhards 2005-07-29 + */ + break; + + if(syncfile) + f->f_flags |= SYNC_FILE; + f->f_un.f_file.bDynamicName = 1; + /* we now allocate a single-byte buffer to create an emtpy previous + * file name. That way, we do not need to check for an uninitialized + * variable when we do the comparison. This saves us some CPU cycles + * on each message processed - and it also simplifies code. If we + * do not get memory, we more or less silently ignore the problem, + * because we can't do anything against it right now. + */ + if((f->f_un.f_file.pCurrName = malloc(sizeof(char))) == NULL) { + f->f_type = F_UNUSED; + dprintf("Could not allocate memory for pCurrName - selector disabled.\n"); + } else { + *f->f_un.f_file.pCurrName = '\0'; + } + break; + case '|': case '/': /* rgerhards 2004-11-17: from now, we need to have different @@ -7961,19 +8132,20 @@ static rsRetVal cfline(char *line, register struct filed *f) */ break; - if (syncfile) + if(syncfile) f->f_flags |= SYNC_FILE; - if (f->f_type == F_PIPE) { - f->f_file = open(f->f_un.f_fname, O_RDWR|O_NONBLOCK); + f->f_un.f_file.bDynamicName = 0; + if(f->f_type == F_PIPE) { + f->f_file = open(f->f_un.f_file.f_fname, O_RDWR|O_NONBLOCK); } else { - f->f_file = open(f->f_un.f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + f->f_file = open(f->f_un.f_file.f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, 0644); } if ( f->f_file < 0 ){ f->f_file = -1; - dprintf("Error opening log file: %s\n", f->f_un.f_fname); - logerror(f->f_un.f_fname); + dprintf("Error opening log file: %s\n", f->f_un.f_file.f_fname); + logerror(f->f_un.f_file.f_fname); break; } if (isatty(f->f_file)) { @@ -9123,7 +9295,7 @@ int main(int argc, char **argv) /* prepare emergency logging system */ consfile.f_type = F_CONSOLE; - strcpy(consfile.f_un.f_fname, ctty); + strcpy(consfile.f_un.f_file.f_fname, ctty); cflineSetTemplateAndIOV(&consfile, " TradFmt"); gethostname(LocalHostName, sizeof(LocalHostName)); if ( (p = strchr(LocalHostName, '.')) ) { diff --git a/template.h b/template.h index 80770895..a23a3ba6 100644 --- a/template.h +++ b/template.h @@ -4,8 +4,8 @@ * begun 2004-11-17 rgerhards */ -#ifdef FEATURE_REGEXP -/* Include regular expressions */ +#ifdef FEATURE_REGEXP +/* Include regular expressions */ #include #endif -- cgit