From 0ed4da17a23db22e18856049ab2814e640f309c1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Jan 2012 18:27:45 +0100 Subject: licensing update --- plugins/imptcp/imptcp.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index e8cffbb0..65fe703c 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -10,24 +10,23 @@ * * File begun on 2010-08-10 by RGerhards * - * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "config.h" #if !defined(HAVE_EPOLL_CREATE) @@ -1047,11 +1046,11 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } -#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) +# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) DBGPRINTF("imptcp uses epoll_create1()\n"); epollfd = epoll_create1(EPOLL_CLOEXEC); if(epollfd < 0 && errno == ENOSYS) -#endif +# endif { DBGPRINTF("imptcp uses epoll_create()\n"); /* reading the docs, the number of epoll events passed to -- cgit From 92f7e549df5a1333f037cf3b290e3087fd9ad5da Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 Jan 2012 09:37:45 +0100 Subject: preparing for 5.8.7 --- ChangeLog | 4 +++- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3ecbc960..45709779 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.8.7 [V5-stable] 2011-??-?? +Version 5.8.7 [V5-stable] 2012-01-17 - bugfix: instabilities when using RFC5424 header fields Thanks to Kaiwang Chen for the patch - bugfix: imuxsock did truncate part of received message if it did not @@ -10,6 +10,8 @@ Version 5.8.7 [V5-stable] 2011-??-?? closes: http://bugzilla.adiscon.com/show_bug.cgi?id=290 Thanks to Tomas Heinrich for the patch - bugfix: stats counter were not properly initialized on creation +- FQDN hostname for multihomed host was not always set to the correct name + if multiple aliases existed. Thanks to Tomas Heinreich for the patch. --------------------------------------------------------------------------- Version 5.8.6 [V5-stable] 2011-10-21 - bugfix: missing whitespace after property-based filter was not detected diff --git a/configure.ac b/configure.ac index 5a77a8a6..6fbc84a6 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.8.6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.8.7],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index 34de4b2b..010ccab6 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

-

This documentation is for version 5.8.6 (v5-stable branch) of rsyslog. +

This documentation is for version 5.8.7 (v5-stable branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

If you like rsyslog, you might -- cgit From 052b1c75945106c6336fd0c172fbaa129248334c Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Mon, 15 Aug 2011 22:22:05 +1000 Subject: add JSON escaping option Following the path taken by the two SQL formatting options, which escape single quotes with double quotes (amongst other things), this patch adds a JSON quoting option. JSON is the opposite to the SQL options, requiring double quotes to be quoted within a string. This patch provides a formatting option implementing this requirement, while piggy-backing on the existing code as much as possible. Signed-off-by: Nathan Scott --- action.c | 2 +- template.c | 90 +++++++++++++++++++++++++++++++++++++++----------------------- template.h | 9 ++++--- 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/action.c b/action.c index 278625ce..6796cf14 100644 --- a/action.c +++ b/action.c @@ -1743,7 +1743,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques } /* check required template options */ if( (iTplOpts & OMSR_RQD_TPL_OPT_SQL) - && (pAction->ppTpl[i]->optFormatForSQL == 0)) { + && (pAction->ppTpl[i]->optFormatEscape == 0)) { errno = 0; errmsg.LogError(0, RS_RET_RQD_TPLOPT_MISSING, "Action disabled. To use this action, you have to specify " "the SQL or stdSQL option in your template!\n"); diff --git a/template.c b/template.c index 2038c6c1..54ca95af 100644 --- a/template.c +++ b/template.c @@ -53,7 +53,12 @@ static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ - +enum { + NO_ESCAPE = 0, + SQL_ESCAPE, + STDSQL_ESCAPE, + JSON_ESCAPE, +}; /* helper to tplToString and strgen's, extends buffer */ #define ALLOC_INC 128 @@ -123,10 +128,12 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * * but they are handled in this way because of legacy (don't break any * existing thing). */ - if(pTpl->optFormatForSQL == 1) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); - else if(pTpl->optFormatForSQL == 2) - doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); + if(pTpl->optFormatEscape == SQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, SQL_ESCAPE); + else if(pTpl->optFormatEscape == JSON_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, JSON_ESCAPE); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) + doEscape(&pVal, &iLenVal, &bMustBeFreed, STDSQL_ESCAPE); } /* got source, now copy over */ if(iLenVal > 0) { /* may be zero depending on property */ @@ -213,27 +220,29 @@ finalize_it: } -/* Helper to doSQLEscape. This is called if doSQLEscape +/* Helper to doEscape. This is called if doEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can * NOT simply return the unmodified string because this * may cause SQL injection. But we also can not simply * abort the run, this would be a DoS. I think an appropriate - * measure is to remove the dangerous \' characters. We + * measure is to remove the dangerous \' characters (SQL). We * replace them by \", which will break the message and * signatures eventually present - but this is the * best thing we can do now (or does anybody * have a better idea?). rgerhards 2004-11-23 - * added support for "escapeMode" (so doSQLEscape for details). - * if mode = 1, then backslashes are changed to slashes. + * added support for escape mode (see doEscape for details). + * if mode = SQL_ESCAPE, then backslashes are changed to slashes. * rgerhards 2005-09-22 */ -static void doSQLEmergencyEscape(register uchar *p, int escapeMode) +static void doEmergencyEscape(register uchar *p, int mode) { while(*p) { - if(*p == '\'') + if((mode == SQL_ESCAPE||mode == STDSQL_ESCAPE) && *p == '\'') *p = '"'; - else if((escapeMode == 1) && (*p == '\\')) + else if((mode == JSON_ESCAPE) && *p == '"') + *p = '\''; + else if((mode == SQL_ESCAPE) && *p == '\\') *p = '/'; ++p; } @@ -258,14 +267,16 @@ static void doSQLEmergencyEscape(register uchar *p, int escapeMode) * smartness depends on config settings. So we add a new option to this * function that allows the caller to select if they want to standard or * "smart" encoding ;) - * new parameter escapeMode is 0 - standard sql, 1 - "smart" engines + * -- + * Parameter "mode" is STDSQL_ESCAPE, SQL_ESCAPE "smart" SQL engines, or + * JSON_ESCAPE for everyone requiring escaped JSON (e.g. ElasticSearch). * 2005-09-22 rgerhards */ rsRetVal -doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode) +doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int mode) { DEFiRet; - uchar *p; + uchar *p = NULL; int iLen; cstr_t *pStrB = NULL; uchar *pszGenerated; @@ -276,26 +287,32 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM assert(pbMustBeFreed != NULL); /* first check if we need to do anything at all... */ - if(escapeMode == 0) + if(mode == STDSQL_ESCAPE) for(p = *pp ; *p && *p != '\'' ; ++p) ; - else + else if(mode == SQL_ESCAPE) for(p = *pp ; *p && *p != '\'' && *p != '\\' ; ++p) ; + else if(mode == JSON_ESCAPE) + for(p = *pp ; *p && *p != '"' ; ++p) + ; /* when we get out of the loop, we are either at the - * string terminator or the first \'. */ - if(*p == '\0') + * string terminator or the first character to escape */ + if(p && *p == '\0') FINALIZE; /* nothing to do in this case! */ p = *pp; iLen = *pLen; CHKiRet(cstrConstruct(&pStrB)); - + while(*p) { - if(*p == '\'') { - CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + if((mode == SQL_ESCAPE || mode == STDSQL_ESCAPE) && *p == '\'') { + CHKiRet(cstrAppendChar(pStrB, (mode == STDSQL_ESCAPE) ? '\'' : '\\')); + iLen++; /* reflect the extra character */ + } else if((mode == SQL_ESCAPE) && *p == '\\') { + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ - } else if((escapeMode == 1) && (*p == '\\')) { + } else if((mode == JSON_ESCAPE) && *p == '"') { CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } @@ -314,7 +331,7 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM finalize_it: if(iRet != RS_RET_OK) { - doSQLEmergencyEscape(*pp, escapeMode); + doEmergencyEscape(*pp, mode); if(pStrB != NULL) cstrDestruct(&pStrB); } @@ -883,11 +900,14 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine) * acknowledged implementing the option. -- rgerhards, 2011-03-21 */ if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) { - pTpl->optFormatForSQL = 2; - DBGPRINTF("strgen suports the stdsql option\n"); + pTpl->optFormatEscape = STDSQL_ESCAPE; + DBGPRINTF("strgen supports the stdsql option\n"); } else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) { - pTpl->optFormatForSQL = 1; - DBGPRINTF("strgen suports the sql option\n"); + pTpl->optFormatEscape = SQL_ESCAPE; + DBGPRINTF("strgen supports the sql option\n"); + } else if(lenMod > 4 && !strcasecmp((char*) szMod+ lenMod - 4, ",json")) { + pTpl->optFormatEscape = JSON_ESCAPE; + DBGPRINTF("strgen supports the json option\n"); } finalize_it: @@ -1015,11 +1035,13 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine) * it anyhow... ;) rgerhards 2004-11-22 */ if(!strcmp(optBuf, "stdsql")) { - pTpl->optFormatForSQL = 2; + pTpl->optFormatEscape = STDSQL_ESCAPE; + } else if(!strcmp(optBuf, "json")) { + pTpl->optFormatEscape = JSON_ESCAPE; } else if(!strcmp(optBuf, "sql")) { - pTpl->optFormatForSQL = 1; + pTpl->optFormatEscape = SQL_ESCAPE; } else if(!strcmp(optBuf, "nosql")) { - pTpl->optFormatForSQL = 0; + pTpl->optFormatEscape = NO_ESCAPE; } else { dbgprintf("Invalid option '%s' ignored.\n", optBuf); } @@ -1180,9 +1202,11 @@ void tplPrintList(void) pTpl = tplRoot; while(pTpl != NULL) { dbgprintf("Template: Name='%s' ", pTpl->pszName == NULL? "NULL" : pTpl->pszName); - if(pTpl->optFormatForSQL == 1) + if(pTpl->optFormatEscape == SQL_ESCAPE) dbgprintf("[SQL-Format (MySQL)] "); - else if(pTpl->optFormatForSQL == 2) + else if(pTpl->optFormatEscape == JSON_ESCAPE) + dbgprintf("[JSON-Escaped Format] "); + else if(pTpl->optFormatEscape == STDSQL_ESCAPE) dbgprintf("[SQL-Format (standard SQL)] "); dbgprintf("\n"); pTpe = pTpl->pEntryRoot; diff --git a/template.h b/template.h index f7ac2e08..b5598b6d 100644 --- a/template.h +++ b/template.h @@ -36,9 +36,10 @@ struct template { int tpenElements; /* number of elements in templateEntry list */ struct templateEntry *pEntryRoot; struct templateEntry *pEntryLast; - char optFormatForSQL; /* in text fields, 0 - do not escape, - * 1 - escape quotes by double quotes, - * 2 - escape "the MySQL way" + char optFormatEscape; /* in text fields, 0 - do not escape, + * 1 - escape "the MySQL way" + * 2 - escape quotes by double quotes, + * 3 - escape double quotes for JSON. */ /* following are options. All are 0/1 defined (either on or off). * we use chars because they are faster than bit fields and smaller @@ -130,7 +131,7 @@ rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); */ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); -rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); +rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); -- cgit From f8019d52f83884acb5e8f8755fe976d1592b4ccb Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Mon, 15 Aug 2011 22:22:13 +1000 Subject: add elasticsearch output module Add support for sending events to elasticsearch - a distributed, RESTful, search engine built on Lucene (www.elasticsearch.org). The output module is enabled via a configure option, and uses libcurl to send the messages from rsyslog to elasticsearch. This patch makes use of the earlier JSON quoting patch to ensure valid JSON strings are sent to the server. Signed-off-by: Nathan Scott --- Makefile.am | 5 + configure.ac | 36 ++++ plugins/omelasticsearch/Makefile.am | 8 + plugins/omelasticsearch/omelasticsearch.c | 271 ++++++++++++++++++++++++++++++ tools/syslogd.c | 3 + 5 files changed, 323 insertions(+) create mode 100644 plugins/omelasticsearch/Makefile.am create mode 100644 plugins/omelasticsearch/omelasticsearch.c diff --git a/Makefile.am b/Makefile.am index de4777b2..fd348a67 100644 --- a/Makefile.am +++ b/Makefile.am @@ -168,6 +168,10 @@ if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif +if ENABLE_ELASTICSEARCH +SUBDIRS += plugins/omelasticsearch +endif + if ENABLE_MMSNMPTRAPD SUBDIRS += plugins/mmsnmptrapd endif @@ -244,5 +248,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-imtemplate \ --enable-omtemplate \ --enable-mmsnmptrapd \ + --enable-elasticsearch \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 6fbc84a6..099b78b9 100644 --- a/configure.ac +++ b/configure.ac @@ -669,6 +669,40 @@ AC_SUBST(SNMP_CFLAGS) AC_SUBST(SNMP_LIBS) +# elasticsearch support +AC_ARG_ENABLE(elasticsearch, + [AS_HELP_STRING([--enable-elasticsearch],[Enable elasticsearch output module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_elasticsearch="yes" ;; + no) enable_elasticsearch="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-elasticsearch) ;; + esac], + [enable_elasticsearch=no] +) +if test "x$enable_elasticsearch" = "xyes"; then + AC_CHECK_PROG( + [HAVE_CURL_CONFIG], + [curl-config], + [yes],,, + ) + if test "x${HAVE_CURL_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([curl-config not found in PATH]) + fi + AC_CHECK_LIB( + [curl], + [curl_global_init], + [CURL_CFLAGS="`curl-config --cflags`" + CURL_LIBS="`curl-config --libs`" + ], + [AC_MSG_FAILURE([curl library is missing])], + [`curl-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ELASTICSEARCH, test x$enable_elasticsearch = xyes) +AC_SUBST(CURL_CFLAGS) +AC_SUBST(CURL_LIBS) + + # GnuTLS support AC_ARG_ENABLE(gnutls, [AS_HELP_STRING([--enable-gnutls],[Enable GNU TLS support @<:@default=no@:>@])], @@ -1205,6 +1239,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ plugins/omudpspoof/Makefile \ + plugins/omelasticsearch/Makefile \ plugins/sm_cust_bindcdr/Makefile \ plugins/mmsnmptrapd/Makefile \ plugins/cust1/Makefile \ @@ -1239,6 +1274,7 @@ echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" echo " omhdfs module will be compiled: $enable_omhdfs" +echo " omelasticsearch module will be compiled: $enable_elasticsearch" echo " omruleset module will be compiled: $enable_omruleset" echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" diff --git a/plugins/omelasticsearch/Makefile.am b/plugins/omelasticsearch/Makefile.am new file mode 100644 index 00000000..a574c72f --- /dev/null +++ b/plugins/omelasticsearch/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omelasticsearch.la + +omelasticsearch_la_SOURCES = omelasticsearch.c +omelasticsearch_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omelasticsearch_la_LDFLAGS = -module -avoid-version +omelasticsearch_la_LIBADD = $(CURL_LIBS) + +EXTRA_DIST = diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c new file mode 100644 index 00000000..ce2e0c1f --- /dev/null +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -0,0 +1,271 @@ +/* omelasticsearch.c + * This is the http://www.elasticsearch.org/ output module. + * + * NOTE: read comments in module-template.h for more specifics! + * + * Copyright 2011 Nathan Scott. + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "statsobj.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP + +/* internal structures */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(statsobj) + +statsobj_t *indexStats; +STATSCOUNTER_DEF(indexConFail, mutIndexConFail) +STATSCOUNTER_DEF(indexSubmit, mutIndexSubmit) +STATSCOUNTER_DEF(indexFailed, mutIndexFailed) +STATSCOUNTER_DEF(indexSuccess, mutIndexSuccess) + +/* REST API for elasticsearch hits this URL: + * http://:// + */ +typedef struct curl_slist HEADER; +typedef struct _instanceData { + CURL *curlHandle; /* libcurl session handle */ + HEADER *postHeader; /* json POST request info */ +} instanceData; + +/* config variables */ +static int restPort = 9200; +static char *hostName = "localhost"; +static char *searchIndex = "system"; +static char *searchType = "events"; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + +BEGINfreeInstance +CODESTARTfreeInstance + if (pData->postHeader) { + curl_slist_free_all(pData->postHeader); + pData->postHeader = NULL; + } + if (pData->curlHandle) { + curl_easy_cleanup(pData->curlHandle); + pData->curlHandle = NULL; + } +ENDfreeInstance + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +rsRetVal +curlPost(instanceData *instance, uchar *message) +{ + CURLcode code; + CURL *curl = instance->curlHandle; + int length = strlen((char *)message); + + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, length); + code = curl_easy_perform(curl); + switch (code) { + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_CONNECT: + case CURLE_WRITE_ERROR: + STATSCOUNTER_INC(indexConFail, mutIndexConFail); + return RS_RET_SUSPENDED; + default: + STATSCOUNTER_INC(indexSubmit, mutIndexSubmit); + return RS_RET_OK; + } +} + +BEGINdoAction +CODESTARTdoAction + CHKiRet(curlPost(pData, ppString[0])); +finalize_it: +ENDdoAction + +/* elasticsearch POST result string ... useful for debugging */ +size_t +curlResult(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + unsigned int i; + char *p = (char *)ptr; + char *jsonData = (char *)userdata; + static char ok[] = "{\"ok\":true,"; + + ASSERT(size == 1); + + if (size == 1 && + nmemb > sizeof(ok)-1 && + strncmp(p, ok, sizeof(ok)-1) == 0) { + STATSCOUNTER_INC(indexSuccess, mutIndexSuccess); + } else { + STATSCOUNTER_INC(indexFailed, mutIndexFailed); + if (Debug) { + DBGPRINTF("omelasticsearch request: %s\n", jsonData); + DBGPRINTF("omelasticsearch result: "); + for (i = 0; i < nmemb; i++) + DBGPRINTF("%c", p[i]); + DBGPRINTF("\n"); + } + } + return size * nmemb; +} + +static rsRetVal +curlSetup(instanceData *instance) +{ + char restURL[2048]; /* libcurl makes a copy, using the stack here is OK */ + HEADER *header; + CURL *handle; + + handle = curl_easy_init(); + if (handle == NULL) { + return RS_RET_OBJ_CREATION_FAILED; + } + + snprintf(restURL, sizeof(restURL)-1, "http://%s:%d/%s/%s", + hostName, restPort, searchIndex, searchType); + header = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8"); + + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult); + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header); + curl_easy_setopt(handle, CURLOPT_URL, restURL); + curl_easy_setopt(handle, CURLOPT_POST, 1); + + instance->curlHandle = handle; + instance->postHeader = header; + + DBGPRINTF("omelasticsearch setup, using REST URL: %s\n", restURL); + return RS_RET_OK; +} + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + if(strncmp((char*) p, ":omelasticsearch:", sizeof(":omelasticsearch:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + p += sizeof(":omelasticsearch:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (uchar*) " StdJSONFmt")); + + /* all good, we can now initialise our private data */ + CHKiRet(curlSetup(pData)); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINmodExit +CODESTARTmodExit + curl_global_cleanup(); + statsobj.Destruct(&indexStats); + objRelease(errmsg, CORE_COMPONENT); + objRelease(statsobj, CORE_COMPONENT); +ENDmodExit + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + restPort = 9200; + hostName = "localhost"; + searchIndex = "system"; + searchType = "events"; + RETiRet; +} + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); + + /* register config file handlers */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchindex", 0, eCmdHdlrGetWord, NULL, &searchIndex, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchtype", 0, eCmdHdlrGetWord, NULL, &searchType, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchhost", 0, eCmdHdlrGetWord, NULL, &hostName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"elasticsearchport", 0, eCmdHdlrInt, NULL, &restPort, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + + if (curl_global_init(CURL_GLOBAL_ALL) != 0) { + errmsg.LogError(0, RS_RET_OBJ_CREATION_FAILED, "CURL fail. -elasticsearch indexing disabled"); + ABORT_FINALIZE(RS_RET_OBJ_CREATION_FAILED); + } + + /* support statistics gathering */ + CHKiRet(statsobj.Construct(&indexStats)); + CHKiRet(statsobj.SetName(indexStats, (uchar *)"elasticsearch")); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"connfail", + ctrType_IntCtr, &indexConFail)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"submits", + ctrType_IntCtr, &indexSubmit)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed", + ctrType_IntCtr, &indexFailed)); + CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"success", + ctrType_IntCtr, &indexSuccess)); + CHKiRet(statsobj.ConstructFinalize(indexStats)); +ENDmodInit + +/* vi:set ai: + */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 0b7bbc96..bec2f9a7 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -334,6 +334,7 @@ static uchar template_WallFmt[] = "\"\r\n\7Message from syslogd@%HOSTNAME% at %t static uchar template_StdUsrMsgFmt[] = "\" %syslogtag%%msg%\n\r\""; static uchar template_StdDBFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-mysql%', '%timegenerated:::date-mysql%', %iut%, '%syslogtag%')\",SQL"; static uchar template_StdPgSQLFmt[] = "\"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%')\",STDSQL"; +static uchar template_StdJSONFmt[] = "\"{\\\"message\\\":\\\"%msg%\\\",\\\"fromhost\\\":\\\"%HOSTNAME%\\\",\\\"facility\\\":\\\"%syslogfacility-text%\\\",\\\"priority\\\":\\\"%syslogpriority-text%\\\",\\\"timereported\\\":\\\"%timereported:::date-rfc3339%\\\",\\\"timegenerated\\\":\\\"%timegenerated:::date-rfc3339%\\\"}\",JSON"; static uchar template_spoofadr[] = "\"%fromhost-ip%\""; /* end templates */ @@ -2173,6 +2174,8 @@ static rsRetVal mainThread() tplAddLine(" StdDBFmt", &pTmp); pTmp = template_StdPgSQLFmt; tplAddLine(" StdPgSQLFmt", &pTmp); + pTmp = template_StdJSONFmt; + tplAddLine(" StdJSONFmt", &pTmp); pTmp = template_spoofadr; tplLastStaticInit(tplAddLine("RSYSLOG_omudpspoofDfltSourceTpl", &pTmp)); -- cgit From 9911dacf59e898bae2c8c6619b85348bf54ddc23 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 Jan 2012 10:42:04 +0100 Subject: elasticsearch: move to asl 2.0 email conversation Nathan/Rainer 2012-01-16&17 --- plugins/omelasticsearch/omelasticsearch.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index ce2e0c1f..3bec1838 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -8,20 +8,19 @@ * * This file is part of rsyslog. * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "config.h" #include "rsyslog.h" -- cgit