diff options
author | Rainer Gerhards <rgerhards@adiscon.com> | 2012-01-17 10:48:39 +0100 |
---|---|---|
committer | Rainer Gerhards <rgerhards@adiscon.com> | 2012-01-17 10:48:39 +0100 |
commit | 37e4aab1ecc63db2b5e72fd8a5df5e4976ae82c8 (patch) | |
tree | 9db8aec809487aebf97889b801bfeb0c477e824c | |
parent | 76c4d6b951453078ab53bf612caf0b8ec9d54bb8 (diff) | |
parent | 9911dacf59e898bae2c8c6619b85348bf54ddc23 (diff) | |
download | rsyslog-37e4aab1ecc63db2b5e72fd8a5df5e4976ae82c8.tar.gz rsyslog-37e4aab1ecc63db2b5e72fd8a5df5e4976ae82c8.tar.xz rsyslog-37e4aab1ecc63db2b5e72fd8a5df5e4976ae82c8.zip |
Merge branch 'v5-stable-elasticsearch' into v5-devel-tmp
Conflicts:
ChangeLog
configure.ac
doc/manual.html
tools/syslogd.c
-rw-r--r-- | ChangeLog | 4 | ||||
-rw-r--r-- | Makefile.am | 5 | ||||
-rw-r--r-- | action.c | 2 | ||||
-rw-r--r-- | configure.ac | 36 | ||||
-rw-r--r-- | plugins/imptcp/imptcp.c | 33 | ||||
-rw-r--r-- | plugins/omelasticsearch/Makefile.am | 8 | ||||
-rw-r--r-- | plugins/omelasticsearch/omelasticsearch.c | 270 | ||||
-rw-r--r-- | template.c | 90 | ||||
-rw-r--r-- | template.h | 9 | ||||
-rw-r--r-- | tools/syslogd.c | 3 |
10 files changed, 404 insertions, 56 deletions
@@ -113,7 +113,7 @@ Version 5.9.0 [V5-DEVEL] (rgerhards), 2011-06-08 affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236 --------------------------------------------------------------------------- -Version 5.8.7 [V5-stable] 2011-??-?? +Version 5.8.7 [V5-stable] 2012-01-17 - bugfix: instabilities when using RFC5424 header fields Thanks to Kaiwang Chen for the patch - bugfix: imuxsock did truncate part of received message if it did not @@ -124,6 +124,8 @@ Version 5.8.7 [V5-stable] 2011-??-?? closes: http://bugzilla.adiscon.com/show_bug.cgi?id=290 Thanks to Tomas Heinrich for the patch - bugfix: stats counter were not properly initialized on creation +- FQDN hostname for multihomed host was not always set to the correct name + if multiple aliases existed. Thanks to Tomas Heinreich for the patch. --------------------------------------------------------------------------- Version 5.8.6 [V5-stable] 2011-10-21 - bugfix: missing whitespace after property-based filter was not detected diff --git a/Makefile.am b/Makefile.am index de4777b2..fd348a67 100644 --- a/Makefile.am +++ b/Makefile.am @@ -168,6 +168,10 @@ if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif +if ENABLE_ELASTICSEARCH +SUBDIRS += plugins/omelasticsearch +endif + if ENABLE_MMSNMPTRAPD SUBDIRS += plugins/mmsnmptrapd endif @@ -244,5 +248,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-imtemplate \ --enable-omtemplate \ --enable-mmsnmptrapd \ + --enable-elasticsearch \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 @@ -1798,7 +1798,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques } /* check required template options */ if( (iTplOpts & OMSR_RQD_TPL_OPT_SQL) - && (pAction->ppTpl[i]->optFormatForSQL == 0)) { + && (pAction->ppTpl[i]->optFormatEscape == 0)) { errno = 0; errmsg.LogError(0, RS_RET_RQD_TPLOPT_MISSING, "Action disabled. To use this action, you have to specify " "the SQL or stdSQL option in your template!\n"); diff --git a/configure.ac b/configure.ac index 98beb121..011905a6 100644 --- a/configure.ac +++ b/configure.ac @@ -671,6 +671,40 @@ AC_SUBST(SNMP_CFLAGS) AC_SUBST(SNMP_LIBS) +# elasticsearch support +AC_ARG_ENABLE(elasticsearch, + [AS_HELP_STRING([--enable-elasticsearch],[Enable elasticsearch output module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_elasticsearch="yes" ;; + no) enable_elasticsearch="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-elasticsearch) ;; + esac], + [enable_elasticsearch=no] +) +if test "x$enable_elasticsearch" = "xyes"; then + AC_CHECK_PROG( + [HAVE_CURL_CONFIG], + [curl-config], + [yes],,, + ) + if test "x${HAVE_CURL_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([curl-config not found in PATH]) + fi + AC_CHECK_LIB( + [curl], + [curl_global_init], + [CURL_CFLAGS="`curl-config --cflags`" + CURL_LIBS="`curl-config --libs`" + ], + [AC_MSG_FAILURE([curl library is missing])], + [`curl-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ELASTICSEARCH, test x$enable_elasticsearch = xyes) +AC_SUBST(CURL_CFLAGS) +AC_SUBST(CURL_LIBS) + + # GnuTLS support AC_ARG_ENABLE(gnutls, [AS_HELP_STRING([--enable-gnutls],[Enable GNU TLS support @<:@default=no@:>@])], @@ -1207,6 +1241,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ plugins/omudpspoof/Makefile \ + plugins/omelasticsearch/Makefile \ plugins/sm_cust_bindcdr/Makefile \ plugins/mmsnmptrapd/Makefile \ plugins/cust1/Makefile \ @@ -1241,6 +1276,7 @@ echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" echo " omhdfs module will be compiled: $enable_omhdfs" +echo " omelasticsearch module will be compiled: $enable_elasticsearch" echo " omruleset module will be compiled: $enable_omruleset" echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 8751637d..d5855879 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -10,24 +10,23 @@ * * File begun on 2010-08-10 by RGerhards * - * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "config.h" #if !defined(HAVE_EPOLL_CREATE) @@ -1132,11 +1131,11 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } -#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) DBGPRINTF("imptcp uses epoll_create1()\n"); epollfd = epoll_create1(EPOLL_CLOEXEC); if(epollfd < 0 && errno == ENOSYS) -#endif +# endif { DBGPRINTF("imptcp uses epoll_create()\n"); /* reading the docs, the number of epoll events passed to diff --git a/plugins/omelasticsearch/Makefile.am b/plugins/omelasticsearch/Makefile.am new file mode 100644 index 00000000..a574c72f --- /dev/null +++ b/plugins/omelasticsearch/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omelasticsearch.la + +omelasticsearch_la_SOURCES = omelasticsearch.c +omelasticsearch_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omelasticsearch_la_LDFLAGS = -module -avoid-version +omelasticsearch_la_LIBADD = $(CURL_LIBS) + +EXTRA_DIST = diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c new file mode 100644 index 00000000..3bec1838 --- /dev/null +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -0,0 +1,270 @@ +/* omelasticsearch.c + * This is the http://www.elasticsearch.org/ output module. + * + * NOTE: read comments in module-template.h for more specifics! + * + * Copyright 2011 Nathan Scott. + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <curl/curl.h> +#include <curl/types.h> +#include <curl/easy.h> +#include <assert.h> +#include <signal.h> +#include <errno.h> +#include <time.h> +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "statsobj.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP + +/* internal structures */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(statsobj) + +statsobj_t *indexStats; +STATSCOUNTER_DEF(indexConFail, mutIndexConFail) +STATSCOUNTER_DEF(indexSubmit, mutIndexSubmit) +STATSCOUNTER_DEF(indexFailed, mutIndexFailed) +STATSCOUNTER_DEF(indexSuccess, mutIndexSuccess) + +/* REST API for elasticsearch hits this URL: + * http://<hostName>:<restPort>/<searchIndex>/<searchType> + */ +typedef struct curl_slist HEADER; +typedef struct _instanceData { + CURL *curlHandle; /* libcurl session handle */ + HEADER *postHeader; /* json POST request info */ +} instanceData; + +/* config variables */ +static int restPort = 9200; +static char *hostName = "localhost"; +static char *searchIndex = "system"; +static char *searchType = "events"; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +BEGINfreeInstance +CODESTARTfreeInstance + if (pData->postHeader) { + curl_slist_free_all(pData->postHeader); + pData->postHeader = NULL; + } + if (pData->curlHandle) { + curl_easy_cleanup(pData->curlHandle); + pData->curlHandle = NULL; + } +ENDfreeInstance + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +rsRetVal +curlPost(instanceData *instance, uchar *message) +{ + CURLcode code; + CURL *curl = instance->curlHandle; + int length = strlen((char *)message); + + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, length); + code = curl_easy_perform(curl); + switch (code) { + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_CONNECT: + case CURLE_WRITE_ERROR: + STATSCOUNTER_INC(indexConFail, mutIndexConFail); + return RS_RET_SUSPENDED; + default: + STATSCOUNTER_INC(indexSubmit, mutIndexSubmit); + return RS_RET_OK; + } +} + +BEGINdoAction +CODESTARTdoAction + CHKiRet(curlPost(pData, ppString[0])); +finalize_it: +ENDdoAction + +/* elasticsearch POST result string ... useful for debugging */ +size_t +curlResult(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + unsigned int i; + char *p = (char *)ptr; + char *jsonData = (char *)userdata; + static char ok[] = "{\"ok\":true,"; + + ASSERT(size == 1); + + if (size == 1 && + nmemb > sizeof(ok)-1 && + strncmp(p, ok, sizeof(ok)-1) == 0) { + STATSCOUNTER_INC(indexSuccess, mutIndexSuccess); + } else { + STATSCOUNTER_INC(indexFailed, mutIndexFailed); + if (Debug) { + DBGPRINTF("omelasticsearch request: %s\n", jsonData); + DBGPRINTF("omelasticsearch result: "); + for (i = 0; i < nmemb; i++) + DBGPRINTF("%c", p[i]); + DBGPRINTF("\n"); + } + } + return size * nmemb; +} + +static rsRetVal +curlSetup(instanceData *instance) +{ + char restURL[2048]; /* libcurl makes a copy, using the stack here is OK */ + HEADER *header; + CURL *handle; + + handle = curl_easy_init(); + if (handle == NULL) { + return RS_RET_OBJ_CREATION_FAILED; + } + + snprintf(restURL, sizeof(restURL)-1, "http://%s:%d/%s/%s", + hostName, restPort, searchIndex, searchType); + header = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8"); + + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult); + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header); + curl_easy_setopt(handle, CURLOPT_URL, restURL); + curl_easy_setopt(handle, CURLOPT_POST, 1); + + instance->curlHandle = handle; + instance->postHeader = header; + + DBGPRINTF("omelasticsearch setup, using REST URL: %s\n", restURL); + return RS_RET_OK; +} + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + if(strncmp((char*) p, ":omelasticsearch:", sizeof(":omelasticsearch:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + p += sizeof(":omelasticsearch:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (uchar*) " StdJSONFmt")); + + /* all good, we can now initialise our private data */ + CHKiRet(curlSetup(pData)); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINmodExit +CODESTARTmodExit + curl_global_cleanup(); + statsobj.Destruct(&indexStats); + objRelease(errmsg, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); +ENDmodExit + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + restPort = 9200; + hostName = "localhost"; + searchIndex = "system"; + searchType = "events"; + RETiRet; +} + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); + + /* register config file handlers */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchindex", 0, eCmdHdlrGetWord, NULL, &searchIndex, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchtype", 0, eCmdHdlrGetWord, NULL, &searchType, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchhost", 0, eCmdHdlrGetWord, NULL, &hostName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchport", 0, eCmdHdlrInt, NULL, &restPort, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + + if (curl_global_init(CURL_GLOBAL_ALL) != 0) { + errmsg.LogError(0, RS_RET_OBJ_CREATION_FAILED, "CURL fail. -elasticsearch indexing disabled"); + ABORT_FINALIZE(RS_RET_OBJ_CREATION_FAILED); + } + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&indexStats)); + CHKiRet(statsobj.SetName(indexStats, (uchar *)"elasticsearch")); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"connfail", + ctrType_IntCtr, &indexConFail)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"submits", + ctrType_IntCtr, &indexSubmit)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed", + ctrType_IntCtr, &indexFailed)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"success", + ctrType_IntCtr, &indexSuccess)); + CHKiRet(statsobj.ConstructFinalize(indexStats)); +ENDmodInit + +/* vi:set ai: + */ @@ -53,7 +53,12 @@ static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ - +enum { + NO_ESCAPE = 0, + SQL_ESCAPE, + STDSQL_ESCAPE, + JSON_ESCAPE, +}; /* helper to tplToString and strgen's, extends buffer */ #define ALLOC_INC 128 @@ -123,10 +128,12 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * * but they are handled in this way because of legacy (don't break any * existing thing). */ - if(pTpl->optFormatForSQL == 1) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); - else if(pTpl->optFormatForSQL == 2) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); + if(pTpl->optFormatEscape == SQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, SQL_ESCAPE); + else if(pTpl->optFormatEscape == JSON_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, JSON_ESCAPE); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, STDSQL_ESCAPE); } /* got source, now copy over */ if(iLenVal > 0) { /* may be zero depending on property */ @@ -213,27 +220,29 @@ finalize_it: } -/* Helper to doSQLEscape. This is called if doSQLEscape +/* Helper to doEscape. This is called if doEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can * NOT simply return the unmodified string because this * may cause SQL injection. But we also can not simply * abort the run, this would be a DoS. I think an appropriate - * measure is to remove the dangerous \' characters. We + * measure is to remove the dangerous \' characters (SQL). We * replace them by \", which will break the message and * signatures eventually present - but this is the * best thing we can do now (or does anybody * have a better idea?). rgerhards 2004-11-23 - * added support for "escapeMode" (so doSQLEscape for details). - * if mode = 1, then backslashes are changed to slashes. + * added support for escape mode (see doEscape for details). + * if mode = SQL_ESCAPE, then backslashes are changed to slashes. * rgerhards 2005-09-22 */ -static void doSQLEmergencyEscape(register uchar *p, int escapeMode) +static void doEmergencyEscape(register uchar *p, int mode) { while(*p) { - if(*p == '\'') + if((mode == SQL_ESCAPE||mode == STDSQL_ESCAPE) && *p == '\'') *p = '"'; - else if((escapeMode == 1) && (*p == '\\')) + else if((mode == JSON_ESCAPE) && *p == '"') + *p = '\''; + else if((mode == SQL_ESCAPE) && *p == '\\') *p = '/'; ++p; } @@ -258,14 +267,16 @@ static void doSQLEmergencyEscape(register uchar *p, int escapeMode) * smartness depends on config settings. So we add a new option to this * function that allows the caller to select if they want to standard or * "smart" encoding ;) - * new parameter escapeMode is 0 - standard sql, 1 - "smart" engines + * -- + * Parameter "mode" is STDSQL_ESCAPE, SQL_ESCAPE "smart" SQL engines, or + * JSON_ESCAPE for everyone requiring escaped JSON (e.g. ElasticSearch). * 2005-09-22 rgerhards */ rsRetVal -doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode) +doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int mode) { DEFiRet; - uchar *p; + uchar *p = NULL; int iLen; cstr_t *pStrB = NULL; uchar *pszGenerated; @@ -276,26 +287,32 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM assert(pbMustBeFreed != NULL); /* first check if we need to do anything at all... */ - if(escapeMode == 0) + if(mode == STDSQL_ESCAPE) for(p = *pp ; *p && *p != '\'' ; ++p) ; - else + else if(mode == SQL_ESCAPE) for(p = *pp ; *p && *p != '\'' && *p != '\\' ; ++p) ; + else if(mode == JSON_ESCAPE) + for(p = *pp ; *p && *p != '"' ; ++p) + ; /* when we get out of the loop, we are either at the - * string terminator or the first \'. */ - if(*p == '\0') + * string terminator or the first character to escape */ + if(p && *p == '\0') FINALIZE; /* nothing to do in this case! */ p = *pp; iLen = *pLen; CHKiRet(cstrConstruct(&pStrB)); - + while(*p) { - if(*p == '\'') { - CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + if((mode == SQL_ESCAPE || mode == STDSQL_ESCAPE) && *p == '\'') { + CHKiRet(cstrAppendChar(pStrB, (mode == STDSQL_ESCAPE) ? '\'' : '\\')); + iLen++; /* reflect the extra character */ + } else if((mode == SQL_ESCAPE) && *p == '\\') { + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ - } else if((escapeMode == 1) && (*p == '\\')) { + } else if((mode == JSON_ESCAPE) && *p == '"') { CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } @@ -314,7 +331,7 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM finalize_it: if(iRet != RS_RET_OK) { - doSQLEmergencyEscape(*pp, escapeMode); + doEmergencyEscape(*pp, mode); if(pStrB != NULL) cstrDestruct(&pStrB); } @@ -883,11 +900,14 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) * acknowledged implementing the option. -- rgerhards, 2011-03-21 */ if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) { - pTpl->optFormatForSQL = 2; - DBGPRINTF("strgen suports the stdsql option\n"); + pTpl->optFormatEscape = STDSQL_ESCAPE; + DBGPRINTF("strgen supports the stdsql option\n"); } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) { - pTpl->optFormatForSQL = 1; - DBGPRINTF("strgen suports the sql option\n"); + pTpl->optFormatEscape = SQL_ESCAPE; + DBGPRINTF("strgen supports the sql option\n"); + } else if(lenMod > 4 && !strcasecmp((char*) szMod+ lenMod - 4, ",json")) { + pTpl->optFormatEscape = JSON_ESCAPE; + DBGPRINTF("strgen supports the json option\n"); } finalize_it: @@ -1015,11 +1035,13 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) * it anyhow... ;) rgerhards 2004-11-22 */ if(!strcmp(optBuf, "stdsql")) { - pTpl->optFormatForSQL = 2; + pTpl->optFormatEscape = STDSQL_ESCAPE; + } else if(!strcmp(optBuf, "json")) { + pTpl->optFormatEscape = JSON_ESCAPE; } else if(!strcmp(optBuf, "sql")) { - pTpl->optFormatForSQL = 1; + pTpl->optFormatEscape = SQL_ESCAPE; } else if(!strcmp(optBuf, "nosql")) { - pTpl->optFormatForSQL = 0; + pTpl->optFormatEscape = NO_ESCAPE; } else { dbgprintf("Invalid option '%s' ignored.\n", optBuf); } @@ -1180,9 +1202,11 @@ void tplPrintList(void) pTpl = tplRoot; while(pTpl != NULL) { dbgprintf("Template: Name='%s' ", pTpl->pszName == NULL? "NULL" : pTpl->pszName); - if(pTpl->optFormatForSQL == 1) + if(pTpl->optFormatEscape == SQL_ESCAPE) dbgprintf("[SQL-Format (MySQL)] "); - else if(pTpl->optFormatForSQL == 2) + else if(pTpl->optFormatEscape == JSON_ESCAPE) + dbgprintf("[JSON-Escaped Format] "); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) dbgprintf("[SQL-Format (standard SQL)] "); dbgprintf("\n"); pTpe = pTpl->pEntryRoot; @@ -36,9 +36,10 @@ struct template { int tpenElements; /* number of elements in templateEntry list */ struct templateEntry *pEntryRoot; struct templateEntry *pEntryLast; - char optFormatForSQL; /* in text fields, 0 - do not escape, - * 1 - escape quotes by double quotes, - * 2 - escape "the MySQL way" + char optFormatEscape; /* in text fields, 0 - do not escape, + * 1 - escape "the MySQL way" + * 2 - escape quotes by double quotes, + * 3 - escape double quotes for JSON. */ /* following are options. All are 0/1 defined (either on or off). * we use chars because they are faster than bit fields and smaller @@ -130,7 +131,7 @@ rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); */ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); -rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); +rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); diff --git a/tools/syslogd.c b/tools/syslogd.c index 65770404..9553ad08 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -335,6 +335,7 @@ static uchar template_StdUsrMsgFmt[] = "\" %syslogtag%%msg%\n\r\""; static uchar template_StdDBFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-mysql%', '%timegenerated:::date-mysql%', %iut%, '%syslogtag%')\",SQL"; static uchar template_StdPgSQLFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%')\",STDSQL"; static uchar template_SysklogdFileFormat[] = "\"%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%\n\""; +static uchar template_StdJSONFmt[] = "\"{\\\"message\\\":\\\"%msg%\\\",\\\"fromhost\\\":\\\"%HOSTNAME%\\\",\\\"facility\\\":\\\"%syslogfacility-text%\\\",\\\"priority\\\":\\\"%syslogpriority-text%\\\",\\\"timereported\\\":\\\"%timereported:::date-rfc3339%\\\",\\\"timegenerated\\\":\\\"%timegenerated:::date-rfc3339%\\\"}\",JSON"; static uchar template_spoofadr[] = "\"%fromhost-ip%\""; /* end templates */ @@ -2183,6 +2184,8 @@ static rsRetVal mainThread() tplAddLine("RSYSLOG_SysklogdFileFormat", &pTmp); pTmp = template_StdPgSQLFmt; tplAddLine(" StdPgSQLFmt", &pTmp); + pTmp = template_StdJSONFmt; + tplAddLine(" StdJSONFmt", &pTmp); pTmp = template_spoofadr; tplLastStaticInit(tplAddLine("RSYSLOG_omudpspoofDfltSourceTpl", &pTmp)); |