diff options
| -rw-r--r-- | doc/Makefile.am | 1 | ||||
| -rw-r--r-- | doc/modules.html | 5 | ||||
| -rw-r--r-- | doc/omoracle.html | 78 | ||||
| -rw-r--r-- | doc/rsyslog_conf_modules.html | 5 | ||||
| -rw-r--r-- | plugins/omoracle/omoracle.c | 285 | ||||
| -rw-r--r-- | plugins/omoracle/omoracle.h | 2 | ||||
| -rw-r--r-- | runtime/rsyslog.h | 1 |
7 files changed, 347 insertions, 30 deletions
diff --git a/doc/Makefile.am b/doc/Makefile.am index 3015d6b5..0f3dca70 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -33,6 +33,7 @@ html_files = \ dev_queue.html \ omsnmp.html \ ommysql.html \ + omoracle.html \ omlibdbi.html \ imfile.html \ imtcp.html \ diff --git a/doc/modules.html b/doc/modules.html index 92887508..4eae6db3 100644 --- a/doc/modules.html +++ b/doc/modules.html @@ -4,9 +4,8 @@ </head> <body> <h1>About rsyslog Modules</h1> - <P><small><i>Written by - <a href="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer - Gerhards</a> (2007-07-28)</i></small></P> +<P><small><i>Written by +<a href="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer Gerhards</a> (2007-07-28)</i></small></P> <p><font color="#FF0000"><b>This document is incomplete. The module interface is also quite incomplete and under development. Do not currently use it!</b></font> You may want to visit <a href="http://rgerhards.blogspot.com/">Rainer's blog</a> diff --git a/doc/omoracle.html b/doc/omoracle.html new file mode 100644 index 00000000..40f6360f --- /dev/null +++ b/doc/omoracle.html @@ -0,0 +1,78 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>Oracle Database Output Module</title> +</head> + +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>Oracle Database Output Module</h1> +<p><b>Module Name: omoracle</b></p> +<p><b>Author: </b>Luis Fernando Muñoz Mejías <Luis.Fernando.Munoz.Mejias@cern.ch></p> +<p><b>Available since: </b>: 4.3.0 +<p><b>Status: </b>: contributed module, not maitained by rsyslog core authors +<p><b>Description</b>:</p> +<p>This module provides native support for logging to Oracle databases. It offers +superior performance over the more generic <a href="omlibdbi.html">omlibdbi</a> module. +It also includes a number of enhancements, most importantly prepared statements and +batching, what provides a big performance improvements. +</p> +<p>Note that this module is maintained by its original author. If you need assistance with it, +it is suggested to post questions to the +<a href="http://lists.adiscon.net/mailman/listinfo/rsyslog">rsyslog mailing list</a>. +<p>From the header comments of this module: +<p><pre> + + This is an output module feeding directly to an Oracle + database. It uses Oracle Call Interface, a propietary module + provided by Oracle. + + Selector lines to be used are of this form: + + :omoracle:;TemplateName + + The module gets its configuration via rsyslog $... directives, + namely: + + $OmoracleDBUser: user name to log in on the database. + + $OmoracleDBPassword: password to log in on the database. + + $OmoracleDB: connection string (an Oracle easy connect or a db + name as specified by tnsnames.ora) + + $OmoracleBatchSize: Number of elements to send to the DB on each + transaction. + + $OmoracleStatement: Statement to be prepared and executed in + batches. Please note that Oracle's prepared statements have their + placeholders as ':identifier', and this module uses the colon to + guess how many placeholders there will be. + + All these directives are mandatory. The dbstring can be an Oracle + easystring or a DB name, as present in the tnsnames.ora file. + + The form of the template is just a list of strings you want + inserted to the DB, for instance: + + $template TestStmt,"%hostname%%msg%" + + Will provide the arguments to a statement like + + $OmoracleStatement \ + insert into foo(hostname,message)values(:host,:message) + + Also note that identifiers to placeholders are arbitrarry. You + need to define the properties on the template in the correct order + you want them passed to the statement! +</pre> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index a281d9e7..df9abeea 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -19,6 +19,7 @@ generic database output module (Firebird/Interbase, MS SQL, Sybase, SQLLite, Ingres, Oracle, mSQL)</li> <li><a href="ommail.html">ommail</a> - permits rsyslog to alert folks by mail if something important happens</li> +<li><a href="omoracle.html">omoracle</a> - output module for Oracle (native OCI interface)</li> <li><a href="imfile.html">imfile</a> - input module for text files</li> <li><a href="imrelp.html">imrelp</a> - RELP @@ -44,9 +45,9 @@ only available if it has been loaded (using $ModLoad).</p> [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL -version 2 or higher.</font></p> +version 3 or higher.</font></p> </body> </html> diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ea910d3a..71cc8e1f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -12,12 +12,36 @@ namely: $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db name as specified by tnsnames.ora) - - All fields are mandatory. The dbstring can be an Oracle easystring - or a DB name, as present in the tnsnames.ora file. + + $OmoracleBatchSize: Number of elements to send to the DB on each + transaction. + + $OmoracleStatement: Statement to be prepared and executed in + batches. Please note that Oracle's prepared statements have their + placeholders as ':identifier', and this module uses the colon to + guess how many placeholders there will be. + + All these directives are mandatory. The dbstring can be an Oracle + easystring or a DB name, as present in the tnsnames.ora file. + + The form of the template is just a list of strings you want + inserted to the DB, for instance: + + $template TestStmt,"%hostname%%msg%" + + Will provide the arguments to a statement like + + $OmoracleStatement \ + insert into foo(hostname,message)values(:host,:message) + + Also note that identifiers to placeholders are arbitrarry. You + need to define the properties on the template in the correct order + you want them passed to the statement! Author: Luis Fernando Muñoz Mejías <Luis.Fernando.Munoz.Mejias@cern.ch> @@ -35,6 +59,7 @@ #include <signal.h> #include <time.h> #include <assert.h> +#include <ctype.h> #include "dirty.h" #include "syslogd-types.h" #include "srUtils.h" @@ -50,6 +75,22 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +/** */ +struct oracle_batch +{ + /* Batch size */ + int size; + /* Last element inserted in the buffer. The batch will be + * executed when n == size */ + int n; + /* Number of arguments the statement takes */ + int arguments; + /* Parameters to pass to the statement on this transaction */ + char*** parameters; + /* Binding parameters */ + OCIBind** bindings; +}; + typedef struct _instanceData { /* Environment handler, the base for any OCI work. */ OCIEnv* environment; @@ -63,10 +104,12 @@ typedef struct _instanceData { OCISvcCtx* service; /* Credentials object for the connection. */ OCIAuthInfo* authinfo; - /* Binding parameters, currently unused */ - OCIBind* binding; /* Connection string, kept here for possible retries. */ char* connection; + /* Statement to be prepared. */ + char* txt_statement; + /* Batch */ + struct oracle_batch batch; } instanceData; /** Database name, to be filled by the $OmoracleDB directive */ @@ -76,6 +119,14 @@ static char* db_name; static char* db_user; /** Database password, to be filled by the $OmoracleDBPassword */ static char* db_password; +/** Batch size. */ +static int batch_size; +/** Statement to prepare and execute */ +static char* db_statement; +/** Whether or not the core supports the newer array interface. The + * module is able to work in both modes, but the newer is the + * recommended one for performance reasons. */ +static int array_passing; /** Generic function for handling errors from OCI. @@ -128,9 +179,90 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_ERROR; } +/** Callback for OCIBindDynamic. + * + * OCI doesn't insert an array of char* by itself (although it can + * handle arrays of int), so we must either run in batches of size one + * (no way) or bind all parameters with OCI_DATA_AT_EXEC instead of + * OCI_DEFAULT, and then give this function as an argument to + * OCIBindDynamic so that it is able to handle all strings in a single + * server trip. + * + * See the documentation of OCIBindDynamic + * (http://download.oracle.com/docs/cd/B28359_01/appdev.111/b28395/oci16rel003.htm#i444015) + * for more details. + */ +static int bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, + int iter, int __attribute__((unused)) idx, + char** out, int* buflen, unsigned char* piece, + void** bd) +{ + *out = in[iter]; + *buflen = strlen(*out) + 1; + dbgprintf ("omoracle bound line %d, length %d: %s\n", iter, *buflen, + *out); + *piece = OCI_ONE_PIECE; + *bd = NULL; + return OCI_CONTINUE; +} + + +/** Returns the number of bind parameters for the statement given as + * an argument. It counts the number of appearances of ':', as in + * + * insert into foo(bar, baz) values(:bar, :baz) + * + * while taking in account that string literals must not be parsed. */ +static int count_bind_parameters(char* p) +{ + int n = 0; + int enable = 1; + + for (; *p; p++) + if (enable && *p == BIND_MARK ) + n++; + else if (*p == '\'') + enable ^= 1; + dbgprintf ("omoracle statement has %d parameters\n", n); + return n; +} + +/** Prepares the statement, binding all its positional parameters */ +static int prepare_statement(instanceData* pData) +{ + int i; + DEFiRet; + + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->txt_statement, + strlen(pData->txt_statement), + OCI_NTV_SYNTAX, OCI_DEFAULT)); + for (i = 0; i < pData->batch.arguments; i++) { + CHECKERR(pData->error, + OCIBindByPos(pData->statement, + pData->batch.bindings+i, + pData->error, i+1, NULL, + MAX_BUFSIZE * + sizeof ***pData->batch.parameters, + SQLT_STR, NULL, NULL, NULL, + 0, 0, OCI_DATA_AT_EXEC)); + CHECKERR(pData->error, + OCIBindDynamic(pData->batch.bindings[i], + pData->error, + pData->batch.parameters[i], + bind_dynamic, NULL, NULL)); + } + +finalize_it: + RETiRet; +} + /* Resource allocation */ BEGINcreateInstance + int i, j; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -147,15 +279,76 @@ CODESTARTcreateInstance CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); + pData->txt_statement = strdup(db_statement); + CHKmalloc(pData->txt_statement); + dbgprintf("omoracle will run stored statement: %s\n", + pData->txt_statement); + + pData->batch.n = 0; + pData->batch.size = batch_size; + pData->batch.arguments = count_bind_parameters(pData->txt_statement); + + /* I know, this can be done with a single malloc() call but this is + * easier to read. :) */ + pData->batch.parameters = calloc(pData->batch.arguments, + sizeof *pData->batch.parameters); + CHKmalloc(pData->batch.parameters); + for (i = 0; i < pData->batch.arguments; i++) { + pData->batch.parameters[i] = calloc(pData->batch.size, + sizeof **pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i]); + for (j = 0; j < pData->batch.size; j++) { + /* Each entry has at most MAX_BUFSIZE bytes + * because OCI doesn't like null-terminated + * strings when operating with batches, and + * the maximum size of each entry must be + * provided when binding + * parameters. MAX_BUFSIZE is long enough for + * usual entries. */ + pData->batch.parameters[i][j] = calloc(MAX_BUFSIZE, + sizeof ***pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i][j]); + } + } + + pData->batch.bindings = calloc(pData->batch.arguments, + sizeof *pData->batch.bindings); + CHKmalloc(pData->batch.bindings); finalize_it: ENDcreateInstance +/* Inserts all stored statements into the database, releasing any + * allocated memory. */ +static int insert_to_db(instanceData* pData) +{ + DEFiRet; + + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + pData->batch.n, 0, NULL, NULL, OCI_DEFAULT)); + + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); + + pData->batch.n = 0; +finalize_it: + dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? + "succeeded" : "did not succeed"); + RETiRet; +} + /** Close the session and free anything allocated by createInstance. */ BEGINfreeInstance + int i, j; CODESTARTfreeInstance +/* Before actually releasing our resources, let's try to commit + * anything pending so that we don't lose any messages. */ + insert_to_db(pData); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->environment, OCI_HTYPE_ENV); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); @@ -163,8 +356,15 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); + free(pData->txt_statement); + for (i = 0; i < pData->batch.arguments; i++) { + for (j = 0; j < pData->batch.size; j++) + free(pData->batch.parameters[i][j]); + free(pData->batch.parameters[i]); + } + free(pData->batch.parameters); + free(pData->batch.bindings); dbgprintf ("omoracle freed all its resources\n"); - RETiRet; ENDfreeInstance @@ -190,7 +390,7 @@ CODESTARTtryResume * ... of course I don't know why Oracle might need a full restart... * rgerhards, 2009-03-26 */ - dbgprintf("Attempting to reconnect to DB server\n"); + dbgprintf("omoracle attempting to reconnect to DB server\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, @@ -198,6 +398,7 @@ CODESTARTtryResume pData->connection, strlen(pData->connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); + CHKiRet(prepare_statement(pData)); finalize_it: ENDtryResume @@ -233,7 +434,6 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature BEGINparseSelectorAct - CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -251,11 +451,12 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); } CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, - OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + OMSR_TPL_AS_ARRAY, " StdFmt")); CHKiRet(createInstance(&pData)); CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); - + CHKiRet(prepare_statement(pData)); + dbgprintf ("omoracle module got all its resources allocated " "and connected to the DB\n"); @@ -263,21 +464,24 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction + int i; + char **params = (char**) ppString[0]; CODESTARTdoAction - dbgprintf("omoracle attempting to execute statement %s\n", *ppString); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, pData->error, *ppString, - strlen(*ppString), OCI_NTV_SYNTAX, - OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, pData->statement, pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); + + if (pData->batch.n == pData->batch.size) { + dbgprintf("omoracle batch size limit hit, sending into DB\n"); + CHKiRet(insert_to_db(pData)); + } + + for (i = 0; i < pData->batch.arguments && params[i]; i++) { + dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); + strncpy(pData->batch.parameters[i][pData->batch.n], params[i], + MAX_BUFSIZE); + CHKmalloc(pData->batch.parameters[i][pData->batch.n]); + } + pData->batch.n++; + finalize_it: - dbgprintf ("omoracle %s at executing statement %s\n", - iRet?"did not succeed":"succeeded", *ppString); -/* Clean credentials to avoid leakage in case of core dump. */ ENDdoAction BEGINmodExit @@ -309,16 +513,35 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, memset(db_password, 0, n); free(db_password); } - db_name = db_user = db_password = NULL; + if (db_statement != NULL) + free(db_statement); + db_name = db_user = db_password = db_statement = NULL; + RETiRet; +} + +/** As I don't find any handler that reads an entire line, I write my + * own. */ +static int get_db_statement(char** line, char** stmt) +{ + DEFiRet; + + while (isspace(**line)) + (*line)++; + dbgprintf ("Config line: %s\n", *line); + *stmt = strdup(*line); + CHKmalloc(*stmt); + dbgprintf ("Statement: %s\n", *stmt); +finalize_it: RETiRet; } BEGINmodInit() + rsRetVal (*supported_options)(unsigned long *pOpts); + unsigned long opts; CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); - /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); @@ -331,4 +554,16 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, eCmdHdlrGetWord, NULL, &db_name, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchsize", 0, + eCmdHdlrInt, NULL, &batch_size, + STD_LOADABLE_MODULE_ID)); + CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); + CHKiRet((*supported_options)(&opts)); + if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) + ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); + + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, + eCmdHdlrCustomHandler, get_db_statement, + &db_statement, STD_LOADABLE_MODULE_ID)); + ENDmodInit diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h index b0e70917..92bcacab 100644 --- a/plugins/omoracle/omoracle.h +++ b/plugins/omoracle/omoracle.h @@ -20,4 +20,6 @@ enum { MAX_BUFSIZE = 2048 }; +#define BIND_MARK ':' + #endif diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 8e376291..026fbbed 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -266,6 +266,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_CREAT_PIPE = -2117, /**< error during pipe creation */ RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ + RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ |