diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/Makefile.am | 4 | ||||
| -rw-r--r-- | tools/omfile.c | 234 | ||||
| -rw-r--r-- | tools/zpipe.c | 254 |
3 files changed, 453 insertions, 39 deletions
diff --git a/tools/Makefile.am b/tools/Makefile.am index e523b854..f0f9afab 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -26,8 +26,10 @@ rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS -sbin_PROGRAMS += rsyslog_diag_hostname msggen +sbin_PROGRAMS += rsyslog_diag_hostname msggen zpipe rsyslog_diag_hostname_SOURCES = gethostn.c +zpipe_SOURCES = zpipe.c +zpipe_LDADD = -lz msggen_SOURCES = msggen.c endif diff --git a/tools/omfile.c b/tools/omfile.c index c7283e4d..62a044d8 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -12,7 +12,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -57,6 +57,8 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "unicode-helper.h" +#include "zlibw.h" MODULE_TYPE_OUTPUT @@ -64,16 +66,30 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +DEFobjCurrIf(zlibw) /* The following structure is a dynafile name cache entry. */ struct s_dynaFileCacheEntry { - uchar *pName; /* name currently open, if dynamic name */ + uchar *pName; /* name currently open, if dynamic name */ short fd; /* name associated with file name in cache */ time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; +/* the output buffer structure */ +// TODO: later make this part of the dynafile cache! +#define OUTBUF_LEN 128 // TODO: make dynamic! +typedef struct { + uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ + size_t lenBuf; /* max size of buffer */ + size_t iBuf; /* current buffer index */ + int fd; /* which file descriptor is this buf for? */ + /* elements for zip writing */ + z_stream zStrm; + char zipBuf[OUTBUF_LEN]; +} outbuf_t; + /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -92,7 +108,8 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ - short fd; /* file descriptor for (current) file */ + short fd; /* file descriptor for (current) file */ + outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -132,7 +149,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - printf("[dynamic]\n\ttemplate='%s'" + dbgprintf("[dynamic]\n\ttemplate='%s'" "\tfile cache size=%d\n" "\tcreate directories: %s\n" "\tfile owner %d, group %d\n" @@ -146,9 +163,9 @@ CODESTARTdbgPrintInstInfo pData->bFailOnChown ? "yes" : "no" ); } else { /* regular file */ - printf("%s", pData->f_fname); + dbgprintf("%s", pData->f_fname); if (pData->fd == -1) - printf(" (unused)"); + dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -321,7 +338,8 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) +static void +dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { ASSERT(pCache != NULL); @@ -356,7 +374,8 @@ finalize_it: * relevant files. Part of Shutdown and HUP processing. * rgerhards, 2008-10-23 */ -static inline void dynaFileFreeCacheEntries(instanceData *pData) +static inline void +dynaFileFreeCacheEntries(instanceData *pData) { register int i; ASSERT(pData != NULL); @@ -562,48 +581,32 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } -/* rgerhards 2004-11-11: write to a file output. This - * will be called for all outputs using file semantics, - * for example also for pipes. +/* physically write the file */ -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal +doPhysWrite(instanceData *pData, int fd, char *pszBuf, size_t lenBuf) { off_t actualFileSize; int iLenWritten; DEFiRet; - ASSERT(pData != NULL); - /* first check if we have a dynamic file name and, if so, - * check if it still is ok or a new file needs to be created - */ - if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - /* create the message based on format specified */ +dbgprintf("doPhysWrite, fd %d, iBuf %d\n", fd, (int) lenBuf); again: /* check if we have a file size limit and, if so, * obey to it. */ if(pData->f_sizeLimit != 0) { - actualFileSize = lseek(pData->fd, 0, SEEK_END); + actualFileSize = lseek(fd, 0, SEEK_END); if(actualFileSize >= pData->f_sizeLimit) { char errMsg[256]; /* for now, we simply disable a file once it is * beyond the maximum size. This is better than having * us aborted by the OS... rgerhards 2005-06-21 */ - (void) close(pData->fd); + (void) close(fd); /* try to resolve the situation */ + // TODO: *doesn't work, will need to use new fd ! if(resolveFileSizeLimit(pData) != 0) { /* didn't work out, so disable... */ snprintf(errMsg, sizeof(errMsg), @@ -622,13 +625,12 @@ again: } } - iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); -//dbgprintf("lenwritten: %d\n", iLenWritten); + iLenWritten = write(fd, pszBuf, lenBuf); if(iLenWritten < 0) { int e = errno; char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); + DBGPRINTF("log file (%d) write error %d: %s\n", fd, e, errStr); /* If a named pipe is full, we suspend this action for a while */ if(pData->fileType == eTypePIPE && e == EAGAIN) @@ -667,9 +669,159 @@ again: errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } } else if (pData->bSyncFile) { - fsync(pData->fd); + fsync(fd); + } + + pData->poBuf->iBuf = 0; + +finalize_it: + RETiRet; +} + + +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + */ +static rsRetVal +doZipWrite(instanceData *pData) +{ + outbuf_t *poBuf; + z_stream strm; + int zRet; /* zlib return state */ + DEFiRet; + assert(pData != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ + strm = poBuf->zStrm; /* another shortcut */ + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + zRet = zlibw.DeflateInit(&strm, 9); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateInit()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateInit() done successfully\n"); + + /* now doing the compression */ + strm.avail_in = poBuf->iBuf; + strm.next_in = (Bytef*) poBuf->pszBuf; + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", strm.avail_in, strm.total_in); + strm.avail_out = OUTBUF_LEN; + strm.next_out = (Bytef*) poBuf->zipBuf; + zRet = zlibw.Deflate(&strm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, strm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - strm.avail_out)); + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + +RUNLOG_STR("deflate() should be done successfully\n"); + + zRet = zlibw.DeflateEnd(&strm); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateEnd() done successfully\n"); + +finalize_it: + RETiRet; +} + + +/* flush the output buffer + */ +static rsRetVal +doFlush(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + + if(pData->poBuf->iBuf == 0) + FINALIZE; /* nothing to write, but make this a valid case */ + + if(1) { // zlib enabled! + CHKiRet(doZipWrite(pData)); + } else { + CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); + } + +finalize_it: + RETiRet; +} + + +/* do the actual write process. This function is to be called once we are ready for writing. + * It will do buffered writes and persist data only when the buffer is full. Note that we must + * be careful to detect when the file handle changed. + * rgerhards, 2009-06-03 + */ +static rsRetVal +doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) +{ + int i; + outbuf_t *poBuf; + DEFiRet; + ASSERT(pData != NULL); + ASSERT(pszBuf != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ +dbgprintf("doWrite, pData->fd %d, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", +pData->fd, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); + + if(pData->fd != poBuf->fd) { + // TODO: more efficient use for dynafiles + CHKiRet(doFlush(pData)); + poBuf->fd = pData->fd; + } + + for(i = 0 ; i < lenBuf ; ++i) { + poBuf->pszBuf[poBuf->iBuf++] = pszBuf[i]; + if(poBuf->iBuf == poBuf->lenBuf) { + CHKiRet(doFlush(pData)); + } + } + +finalize_it: + RETiRet; +} + + +/* rgerhards 2004-11-11: write to a file output. This + * will be called for all outputs using file semantics, + * for example also for pipes. + */ +static rsRetVal +writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +{ + DEFiRet; + + ASSERT(pData != NULL); + + /* first check if we have a dynamic file name and, if so, + * check if it still is ok or a new file needs to be created + */ + if(pData->bDynamicName) { + if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + } + + if(pData->fd == -1) { + rsRetVal iRetLocal; + iRetLocal = prepareFile(pData, pData->f_fname); + if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } + /* create the message based on format specified */ + CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); + finalize_it: RETiRet; } @@ -678,15 +830,20 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->fd = -1; + CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); + pData->poBuf->lenBuf = OUTBUF_LEN; +finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance + doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->fd != -1) close(pData->fd); + free(pData->poBuf); ENDfreeInstance @@ -696,7 +853,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->f_fname); + DBGPRINTF("file to log to: %s\n", pData->f_fname); iRet = writeFile(ppString, iMsgOpts, pData); ENDdoAction @@ -875,8 +1032,8 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit - if(pszTplName != NULL) - free(pszTplName); + objRelease(zlibw, LM_ZLIBW_FILENAME); + free(pszTplName); ENDmodExit @@ -891,6 +1048,7 @@ BEGINmodInit(File) CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); diff --git a/tools/zpipe.c b/tools/zpipe.c new file mode 100644 index 00000000..bde6c5c1 --- /dev/null +++ b/tools/zpipe.c @@ -0,0 +1,254 @@ +/* zpipe.c: example of proper use of zlib's inflate() and deflate() + Not copyrighted -- provided to the public domain + Version 1.5 11 December 2005 Mark Adler + Version 2.0 03 June 2009 Rainer Gerhards */ + +/* RSYSLOG NOTE: + * This file is beeing distributed as part of rsyslog, but is just an + * add-on. Most importantly, rsyslog's copyright does not apply but + * rather the (non-) copyright stated above. + */ + +/* Version history: + 1.0 30 Oct 2004 First version + 1.1 8 Nov 2004 Add void casting for unused return values + Use switch statement for inflate() return values + 1.2 9 Nov 2004 Add assertions to document zlib guarantees + 1.3 6 Apr 2005 Remove incorrect assertion in inf() + 1.4 11 Dec 2005 Add hack to avoid MSDOS end-of-line conversions + Avoid some compiler warnings for input and output buffers + 2.0 03 Jun 2009 Add hack to support multiple deflate records inside a single + file on inflate. This is needed in order to support reading + files created by rsyslog's zip output writer. + */ + +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include "zlib.h" + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include <fcntl.h> +# include <io.h> +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 + +/* Compress from file source to file dest until EOF on source. + def() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_STREAM_ERROR if an invalid compression + level is supplied, Z_VERSION_ERROR if the version of zlib.h and the + version of the library linked do not match, or Z_ERRNO if there is + an error reading or writing the files. */ +int def(FILE *source, FILE *dest, int level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, level); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + flush = feof(source) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + return Z_OK; +} + + +/* initialize stream for deflating (we need this in case of + * multiple records. + * rgerhards, 2009-06-03 + */ +int doInflateInit(z_stream *strm) +{ + int ret; + + /* allocate inflate state */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + strm->avail_in = 0; + strm->next_in = Z_NULL; + ret = inflateInit(strm); + return ret; +} + + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ +int inf(FILE *source, FILE *dest) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + int len; + unsigned char *next_in_save; + unsigned char out[CHUNK]; + + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + len = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + if (len == 0) { + break; + } + strm.avail_in = len; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + strm.avail_out = CHUNK; + strm.next_out = out; + do { + /* fprintf(stderr, "---inner LOOP---, avail_in %d, avail_out %d Byte 0: %x, 1: %x\n", strm.avail_in, strm.avail_out, *strm.next_in, *(strm.next_in+1));*/ + do { + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + /* handle the case that more than one deflate record is contained + * in a single file. -- rgerhards, 2009-06-03 + */ + if(ret == Z_STREAM_END) { + len -= strm.total_in; + if(len > 0) { + next_in_save = strm.next_in; + (void)inflateEnd(&strm); + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + strm.avail_in = len; + strm.next_in = next_in_save; + strm.avail_out = CHUNK; + strm.next_out = out; + ret = Z_OK; /* continue outer loop */ + } + } + } while (strm.avail_in > 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + +/* report a zlib or i/o error */ +void zerr(int ret) +{ + fputs("zpipe: ", stdout); + switch (ret) { + case Z_ERRNO: + if (ferror(stdin)) + fputs("error reading stdin\n", stdout); + if (ferror(stdout)) + fputs("error writing stdout\n", stdout); + break; + case Z_STREAM_ERROR: + fputs("invalid compression level\n", stdout); + break; + case Z_DATA_ERROR: + fputs("invalid or incomplete deflate data\n", stdout); + break; + case Z_MEM_ERROR: + fputs("out of memory\n", stdout); + break; + case Z_VERSION_ERROR: + fputs("zlib version mismatch!\n", stdout); + } +} + +/* compress or decompress from stdin to stdout */ +int main(int argc, char **argv) +{ + int ret; + + /* avoid end-of-line conversions */ + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + + /* do compression if no arguments */ + if (argc == 1) { + ret = def(stdin, stdout, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* do decompression if -d specified */ + else if (argc == 2 && strcmp(argv[1], "-d") == 0) { + ret = inf(stdin, stdout); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* otherwise, report usage */ + else { + fputs("zpipe usage: zpipe [-d] < source > dest\n", stdout); + return 1; + } +} |