summaryrefslogtreecommitdiffstats
path: root/runtime
diff options
context:
space:
mode:
Diffstat (limited to 'runtime')
-rw-r--r--runtime/Makefile.am30
-rw-r--r--runtime/apc.c25
-rw-r--r--runtime/atomic.h128
-rw-r--r--runtime/batch.h205
-rw-r--r--runtime/cfsysline.c3
-rw-r--r--runtime/conf.c52
-rw-r--r--runtime/conf.h10
-rw-r--r--runtime/datetime.c19
-rw-r--r--runtime/datetime.h6
-rw-r--r--runtime/debug.c108
-rw-r--r--runtime/debug.h9
-rw-r--r--runtime/errmsg.h1
-rw-r--r--runtime/expr.h1
-rw-r--r--runtime/glbl.c101
-rw-r--r--runtime/glbl.h12
-rw-r--r--runtime/hashtable.c323
-rw-r--r--runtime/hashtable.h202
-rw-r--r--runtime/hashtable/Makefile26
-rw-r--r--runtime/hashtable/README11
-rw-r--r--runtime/hashtable/hashtable_utility.c71
-rw-r--r--runtime/hashtable/hashtable_utility.h55
-rw-r--r--runtime/hashtable/tester.c270
-rw-r--r--runtime/hashtable_itr.c190
-rw-r--r--runtime/hashtable_itr.h112
-rw-r--r--runtime/hashtable_private.h86
-rw-r--r--runtime/module-template.h108
-rw-r--r--runtime/modules.c213
-rw-r--r--runtime/modules.h44
-rw-r--r--runtime/msg.c592
-rw-r--r--runtime/msg.h44
-rw-r--r--runtime/net.c74
-rw-r--r--runtime/net.h7
-rw-r--r--runtime/netstrms.c4
-rw-r--r--runtime/nsd.h9
-rw-r--r--runtime/nsd_gtls.c11
-rw-r--r--runtime/nsd_ptcp.c23
-rw-r--r--runtime/nsdpoll_ptcp.c290
-rw-r--r--runtime/nsdpoll_ptcp.h60
-rw-r--r--runtime/nsdsel_gtls.c1
-rw-r--r--runtime/nspoll.c195
-rw-r--r--runtime/nspoll.h65
-rw-r--r--runtime/nssel.c1
-rw-r--r--runtime/obj.c14
-rw-r--r--runtime/objomsr.c30
-rw-r--r--runtime/objomsr.h6
-rw-r--r--runtime/parser.c477
-rw-r--r--runtime/parser.h57
-rw-r--r--runtime/prop.c2
-rw-r--r--runtime/queue.c2096
-rw-r--r--runtime/queue.h95
-rw-r--r--runtime/regexp.c1
-rw-r--r--runtime/rsyslog.c29
-rw-r--r--runtime/rsyslog.h170
-rw-r--r--runtime/rule.c99
-rw-r--r--runtime/rule.h7
-rw-r--r--runtime/ruleset.c223
-rw-r--r--runtime/ruleset.h10
-rw-r--r--runtime/sd-daemon.c435
-rw-r--r--runtime/sd-daemon.h261
-rw-r--r--runtime/srUtils.h17
-rw-r--r--runtime/srutils.c33
-rw-r--r--runtime/statsobj.c315
-rw-r--r--runtime/statsobj.h124
-rw-r--r--runtime/stream.c195
-rw-r--r--runtime/stream.h22
-rw-r--r--runtime/strgen.c279
-rw-r--r--runtime/strgen.h60
-rw-r--r--runtime/stringbuf.c10
-rw-r--r--runtime/strmsrv.c3
-rw-r--r--runtime/sync.c3
-rw-r--r--runtime/syslogd-types.h5
-rw-r--r--runtime/sysvar.c4
-rw-r--r--runtime/typedefs.h148
-rw-r--r--runtime/unlimited_select.h1
-rw-r--r--runtime/vm.c18
-rw-r--r--runtime/wti.c362
-rw-r--r--runtime/wti.h30
-rw-r--r--runtime/wtp.c444
-rw-r--r--runtime/wtp.h55
-rw-r--r--runtime/zlibw.c1
80 files changed, 7564 insertions, 2374 deletions
diff --git a/runtime/Makefile.am b/runtime/Makefile.am
index c1a15198..09cb6b41 100644
--- a/runtime/Makefile.am
+++ b/runtime/Makefile.am
@@ -7,8 +7,10 @@ pkglib_LTLIBRARIES =
librsyslog_la_SOURCES = \
rsyslog.c \
rsyslog.h \
+ typedefs.h \
unicode-helper.h \
atomic.h \
+ batch.h \
syslogd-types.h \
module-template.h \
obj-types.h \
@@ -20,6 +22,8 @@ librsyslog_la_SOURCES = \
conf.h \
parser.h \
parser.c \
+ strgen.h \
+ strgen.c \
msg.c \
msg.h \
linkedlist.c \
@@ -42,6 +46,8 @@ librsyslog_la_SOURCES = \
modules.h \
apc.c \
apc.h \
+ statsobj.c \
+ statsobj.h \
sync.c \
sync.h \
expr.c \
@@ -78,6 +84,8 @@ librsyslog_la_SOURCES = \
prop.h \
cfsysline.c \
cfsysline.h \
+ sd-daemon.c \
+ sd-daemon.h \
\
\
../action.h \
@@ -88,6 +96,12 @@ librsyslog_la_SOURCES = \
../parse.c \
../parse.h \
\
+ hashtable.c \
+ hashtable.h \
+ hashtable_itr.c \
+ hashtable_itr.h \
+ hashtable_private.h \
+ \
../outchannel.c \
../outchannel.h \
../template.c \
@@ -96,9 +110,9 @@ librsyslog_la_SOURCES = \
# runtime or will no longer be needed. -- rgerhards, 2008-06-13
if WITH_MODDIRS
-librsyslog_la_CPPFLAGS = -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS)
+librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS)
else
-librsyslog_la_CPPFLAGS = -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS)
+librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS)
endif
#librsyslog_la_LDFLAGS = -module -avoid-version
librsyslog_la_LIBADD = $(DL_LIBS) $(RT_LIBS)
@@ -136,7 +150,10 @@ lmnet_la_LDFLAGS = -module -avoid-version
lmnet_la_LIBADD =
# network stream master class and stream factory
-lmnetstrms_la_SOURCES = netstrms.c netstrms.h netstrm.c netstrm.h nssel.c nssel.h
+lmnetstrms_la_SOURCES = netstrms.c netstrms.h \
+ netstrm.c netstrm.h \
+ nssel.c nssel.h \
+ nspoll.c nspoll.h
lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS)
lmnetstrms_la_LDFLAGS = -module -avoid-version
lmnetstrms_la_LIBADD =
@@ -152,7 +169,9 @@ lmstrmsrv_la_LIBADD =
# plain tcp driver - main driver
pkglib_LTLIBRARIES += lmnsd_ptcp.la
-lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h nsdsel_ptcp.c nsdsel_ptcp.h
+lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h \
+ nsdsel_ptcp.c nsdsel_ptcp.h \
+ nsdpoll_ptcp.c nsdpoll_ptcp.h
lmnsd_ptcp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS)
lmnsd_ptcp_la_LDFLAGS = -module -avoid-version
lmnsd_ptcp_la_LIBADD =
@@ -169,3 +188,6 @@ lmnsd_gtls_la_LDFLAGS = -module -avoid-version
lmnsd_gtls_la_LIBADD = $(GNUTLS_LIBS)
endif
+update-systemd:
+ curl http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c > sd-daemon.c
+ curl http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h > sd-daemon.h
diff --git a/runtime/apc.c b/runtime/apc.c
index bc330e39..3c6b7ec4 100644
--- a/runtime/apc.c
+++ b/runtime/apc.c
@@ -40,9 +40,11 @@
#include "obj.h"
#include "apc.h"
#include "srUtils.h"
+#include "datetime.h"
/* static data */
DEFobjStaticHelpers
+DEFobjCurrIf(datetime)
/* following is a used to implement a monotonically increasing id for the apcs. That
* ID can be used to cancel an apc request. Note that the ID is generated with modulo
@@ -200,7 +202,7 @@ unlistCurrent(apc_list_t **ppList)
DEFiRet;
assert(ppList != NULL);
- time(&tCurr);
+ datetime.GetTime(&tCurr);
if(apcListRoot == NULL || apcListRoot->pApc->ttExec > tCurr) {
*ppList = NULL;
@@ -249,12 +251,11 @@ execScheduled(void)
apc_list_t *pExecList;
apc_list_t *pCurr;
apc_list_t *pNext;
- DEFVARS_mutexProtection_uncond;
DEFiRet;
- BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex);
+ d_pthread_mutex_lock(&listMutex);
iRet = unlistCurrent(&pExecList);
- END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex);
+ d_pthread_mutex_unlock(&listMutex);
CHKiRet(iRet);
if(pExecList != NULL) {
@@ -290,14 +291,12 @@ ENDobjConstruct(apc)
static rsRetVal
apcConstructFinalize(apc_t *pThis, apc_id_t *pID)
{
- DEFVARS_mutexProtection_uncond;
DEFiRet;
ISOBJ_TYPE_assert(pThis, apc);
assert(pID != NULL);
- BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex);
+ d_pthread_mutex_lock(&listMutex);
insertApc(pThis, pID);
- END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex);
-RUNLOG_STR("apcConstructFinalize post mutex unlock\n");
+ d_pthread_mutex_unlock(&listMutex);
RETiRet;
}
@@ -333,12 +332,10 @@ SetParam2(apc_t *pThis, void *param2)
static rsRetVal
CancelApc(apc_id_t id)
{
- DEFVARS_mutexProtection_uncond;
-
BEGINfunc
- BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex);
+ d_pthread_mutex_lock(&listMutex);
deleteApc(id);
- END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex);
+ d_pthread_mutex_unlock(&listMutex);
ENDfunc
return RS_RET_OK;
}
@@ -380,7 +377,7 @@ ENDobjQueryInterface(apc)
* rgerhards, 2009-04-06
*/
BEGINObjClassExit(apc, OBJ_IS_CORE_MODULE) /* class, version */
- //objRelease(apcstk, CORE_COMPONENT);
+ objRelease(datetime, CORE_COMPONENT);
pthread_mutex_destroy(&listMutex);
ENDObjClassExit(apc)
@@ -391,7 +388,7 @@ ENDObjClassExit(apc)
*/
BEGINObjClassInit(apc, 1, OBJ_IS_CORE_MODULE) /* class, version */
/* request objects we use */
- //CHKiRet(objUse(apcstk, CORE_COMPONENT));
+ CHKiRet(objUse(datetime, CORE_COMPONENT));
/* set our own handlers */
OBJSetMethodHandler(objMethod_DEBUGPRINT, apcDebugPrint);
diff --git a/runtime/atomic.h b/runtime/atomic.h
index fc3e0b2d..3e2c0d18 100644
--- a/runtime/atomic.h
+++ b/runtime/atomic.h
@@ -33,22 +33,28 @@
*/
#ifndef INCLUDED_ATOMIC_H
#define INCLUDED_ATOMIC_H
+#include <time.h>
+#include "typedefs.h"
/* for this release, we disable atomic calls because there seem to be some
* portability problems and we can not fix that without destabilizing the build.
* They simply came in too late. -- rgerhards, 2008-04-02
*/
#ifdef HAVE_ATOMIC_BUILTINS
+# define ATOMIC_SUB(data, val, phlpmut) __sync_fetch_and_sub(data, val)
+# define ATOMIC_ADD(data, val) __sync_fetch_and_add(&(data), val)
# define ATOMIC_INC(data, phlpmut) ((void) __sync_fetch_and_add(data, 1))
-# define ATOMIC_INC_AND_FETCH(data) __sync_fetch_and_add(&(data), 1)
+# define ATOMIC_INC_AND_FETCH_int(data, phlpmut) __sync_fetch_and_add(data, 1)
+# define ATOMIC_INC_AND_FETCH_unsigned(data, phlpmut) __sync_fetch_and_add(data, 1)
# define ATOMIC_DEC(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1))
# define ATOMIC_DEC_AND_FETCH(data, phlpmut) __sync_sub_and_fetch(data, 1)
-# define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff))
+# define ATOMIC_FETCH_32BIT(data, phlpmut) ((unsigned) __sync_fetch_and_and(data, 0xffffffff))
# define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1)
# define ATOMIC_STORE_0_TO_INT(data, phlpmut) __sync_fetch_and_and(data, 0)
# define ATOMIC_STORE_1_TO_INT(data, phlpmut) __sync_fetch_and_or(data, 1)
# define ATOMIC_STORE_INT_TO_INT(data, val) __sync_fetch_and_or(&(data), (val))
-# define ATOMIC_CAS(data, oldVal, newVal) __sync_bool_compare_and_swap(&(data), (oldVal), (newVal));
+# define ATOMIC_CAS(data, oldVal, newVal, phlpmut) __sync_bool_compare_and_swap(data, (oldVal), (newVal))
+# define ATOMIC_CAS_time_t(data, oldVal, newVal, phlpmut) __sync_bool_compare_and_swap(data, (oldVal), (newVal))
# define ATOMIC_CAS_VAL(data, oldVal, newVal, phlpmut) __sync_val_compare_and_swap(data, (oldVal), (newVal));
/* functions below are not needed if we have atomics */
@@ -77,18 +83,47 @@
}
# define ATOMIC_STORE_0_TO_INT(data, hlpmut) { \
- pthread_mutex_lock(&hlpmut); \
+ pthread_mutex_lock(hlpmut); \
*(data) = 0; \
- pthread_mutex_unlock(&hlpmut); \
+ pthread_mutex_unlock(hlpmut); \
}
# define ATOMIC_STORE_1_TO_INT(data, hlpmut) { \
- pthread_mutex_lock(&hlpmut); \
+ pthread_mutex_lock(hlpmut); \
*(data) = 1; \
- pthread_mutex_unlock(&hlpmut); \
+ pthread_mutex_unlock(hlpmut); \
}
static inline int
+ ATOMIC_CAS(int *data, int oldVal, int newVal, pthread_mutex_t *phlpmut) {
+ int bSuccess;
+ pthread_mutex_lock(phlpmut);
+ if(*data == oldVal) {
+ *data = newVal;
+ bSuccess = 1;
+ } else {
+ bSuccess = 0;
+ }
+ pthread_mutex_unlock(phlpmut);
+ return(bSuccess);
+ }
+
+ static inline int
+ ATOMIC_CAS_time_t(time_t *data, time_t oldVal, time_t newVal, pthread_mutex_t *phlpmut) {
+ int bSuccess;
+ pthread_mutex_lock(phlpmut);
+ if(*data == oldVal) {
+ *data = newVal;
+ bSuccess = 1;
+ } else {
+ bSuccess = 0;
+ }
+ pthread_mutex_unlock(phlpmut);
+ return(bSuccess);
+ }
+
+
+ static inline int
ATOMIC_CAS_VAL(int *data, int oldVal, int newVal, pthread_mutex_t *phlpmut) {
int val;
pthread_mutex_lock(phlpmut);
@@ -107,6 +142,24 @@
}
static inline int
+ ATOMIC_INC_AND_FETCH_int(int *data, pthread_mutex_t *phlpmut) {
+ int val;
+ pthread_mutex_lock(phlpmut);
+ val = ++(*data);
+ pthread_mutex_unlock(phlpmut);
+ return(val);
+ }
+
+ static inline unsigned
+ ATOMIC_INC_AND_FETCH_unsigned(unsigned *data, pthread_mutex_t *phlpmut) {
+ unsigned val;
+ pthread_mutex_lock(phlpmut);
+ val = ++(*data);
+ pthread_mutex_unlock(phlpmut);
+ return(val);
+ }
+
+ static inline int
ATOMIC_DEC_AND_FETCH(int *data, pthread_mutex_t *phlpmut) {
int val;
pthread_mutex_lock(phlpmut);
@@ -114,18 +167,65 @@
pthread_mutex_unlock(phlpmut);
return(val);
}
-#if 0
-# warning "atomic builtins not available, using nul operations - rsyslogd will probably be racy!"
-# define ATOMIC_INC_AND_FETCH(data) (++(data))
-# define ATOMIC_FETCH_32BIT(data) (data) // TODO: del
-# define ATOMIC_STORE_1_TO_32BIT(data) (data) = 1 // TODO: del
-#endif
+
+ static inline int
+ ATOMIC_FETCH_32BIT(int *data, pthread_mutex_t *phlpmut) {
+ int val;
+ pthread_mutex_lock(phlpmut);
+ val = (*data);
+ pthread_mutex_unlock(phlpmut);
+ return(val);
+ }
+
+ static inline void
+ ATOMIC_SUB(int *data, int val, pthread_mutex_t *phlpmut) {
+ pthread_mutex_lock(phlpmut);
+ (*data) -= val;
+ pthread_mutex_unlock(phlpmut);
+ }
# define DEF_ATOMIC_HELPER_MUT(x) pthread_mutex_t x
# define INIT_ATOMIC_HELPER_MUT(x) pthread_mutex_init(&(x), NULL)
-# define DESTROY_ATOMIC_HELPER_MUT(x) pthread_mutex_init(&(x), NULL)
+# define DESTROY_ATOMIC_HELPER_MUT(x) pthread_mutex_destroy(&(x))
# define PREFER_ATOMIC_INC(data) ((void) ++data)
#endif
+/* we need to handle 64bit atomics seperately as some platforms have
+ * 32 bit atomics, but not 64 biot ones... -- rgerhards, 2010-12-01
+ */
+#ifdef HAVE_ATOMIC_BUILTINS_64BIT
+# define ATOMIC_INC_uint64(data, phlpmut) ((void) __sync_fetch_and_add(data, 1))
+# define ATOMIC_DEC_unit64(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1))
+# define ATOMIC_INC_AND_FETCH_uint64(data, phlpmut) __sync_fetch_and_add(data, 1)
+
+# define DEF_ATOMIC_HELPER_MUT64(x)
+# define INIT_ATOMIC_HELPER_MUT64(x)
+# define DESTROY_ATOMIC_HELPER_MUT64(x)
+#else
+# define ATOMIC_INC_uint64(data, phlpmut) { \
+ pthread_mutex_lock(phlpmut); \
+ ++(*(data)); \
+ pthread_mutex_unlock(phlpmut); \
+ }
+# define ATOMIC_DEC_uint64(data, phlpmut) { \
+ pthread_mutex_lock(phlpmut); \
+ --(*(data)); \
+ pthread_mutex_unlock(phlpmut); \
+ }
+
+ static inline unsigned
+ ATOMIC_INC_AND_FETCH_uint64(uint64 *data, pthread_mutex_t *phlpmut) {
+ uint64 val;
+ pthread_mutex_lock(phlpmut);
+ val = ++(*data);
+ pthread_mutex_unlock(phlpmut);
+ return(val);
+ }
+
+# define DEF_ATOMIC_HELPER_MUT64(x) pthread_mutex_t x
+# define INIT_ATOMIC_HELPER_MUT64(x) pthread_mutex_init(&(x), NULL)
+# define DESTROY_ATOMIC_HELPER_MUT64(x) pthread_mutex_destroy(&(x))
+#endif /* #ifdef HAVE_ATOMIC_BUILTINS_64BIT */
+
#endif /* #ifndef INCLUDED_ATOMIC_H */
diff --git a/runtime/batch.h b/runtime/batch.h
new file mode 100644
index 00000000..944889bd
--- /dev/null
+++ b/runtime/batch.h
@@ -0,0 +1,205 @@
+/* Definition of the batch_t data structure.
+ * I am not sure yet if this will become a full-blown object. For now, this header just
+ * includes the object definition and is not accompanied by code.
+ *
+ * Copyright 2009 by Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+
+#ifndef BATCH_H_INCLUDED
+#define BATCH_H_INCLUDED
+
+#include <string.h>
+#include "msg.h"
+
+/* enum for batch states. Actually, we violate a layer here, in that we assume that a batch is used
+ * for action processing. So far, this seems acceptable, the status is simply ignored inside the
+ * main message queue. But over time, it could potentially be useful to split the two.
+ * rgerhad, 2009-05-12
+ */
+typedef enum {
+ BATCH_STATE_RDY = 0, /* object ready for processing */
+ BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */
+ BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unknown */
+ BATCH_STATE_COMM = 3, /* message successfully commited */
+ BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */
+} batch_state_t;
+
+
+/* an object inside a batch, including any information (state!) needed for it to "life".
+ */
+struct batch_obj_s {
+ obj_t *pUsrp; /* pointer to user object (most often message) */
+ batch_state_t state; /* associated state */
+ /* work variables for action processing; these are reused for each action (or block of
+ * actions)
+ */
+ sbool bFilterOK; /* work area for filter processing (per action, reused!) */
+ sbool bPrevWasSuspended;
+ /* following are caches to save allocs if not absolutely necessary */
+ uchar *staticActStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /**< for strings */
+ /* a cache to save malloc(), if not absolutely necessary */
+ void *staticActParams[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /**< for anything else */
+ size_t staticLenStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE];
+ /* and the same for the message length (if used) */
+ /* end action work variables */
+};
+
+/* the batch
+ * This object is used to dequeue multiple user pointers which are than handed over
+ * to processing. The size of elements is fixed after queue creation, but may be
+ * modified by config variables (better said: queue properties).
+ * Note that a "user pointer" in rsyslog context so far always is a message
+ * object. We stick to the more generic term because queues may potentially hold
+ * other types of objects, too.
+ * rgerhards, 2009-05-12
+ * Note that nElem is not necessarily equal to nElemDeq. This is the case when we
+ * discard some elements (because of configuration) during dequeue processing. As
+ * all Elements are only deleted when the batch is processed, we can not immediately
+ * delete them. So we need to keep their number that we can delete them when the batch
+ * is completed (else, the whole process does not work correctly).
+ */
+struct batch_s {
+ int maxElem; /* maximum number of elements that this batch supports */
+ int nElem; /* actual number of element in this entry */
+ int nElemDeq; /* actual number of elements dequeued (and thus to be deleted) - see comment above! */
+ int iDoneUpTo; /* all messages below this index have state other than RDY */
+ qDeqID deqID; /* ID of dequeue operation that generated this batch */
+ int *pbShutdownImmediate;/* end processing of this batch immediately if set to 1 */
+ sbool bSingleRuleset; /* do all msgs of this batch use a single ruleset? */
+ batch_obj_t *pElem; /* batch elements */
+};
+
+
+/* some inline functions (we may move this off to an object .. or not) */
+static inline void
+batchSetSingleRuleset(batch_t *pBatch, sbool val) {
+ pBatch->bSingleRuleset = val;
+}
+
+/* get the batches ruleset (if we have a single ruleset) */
+static inline ruleset_t*
+batchGetRuleset(batch_t *pBatch) {
+ return (pBatch->nElem > 0) ? ((msg_t*) pBatch->pElem[0].pUsrp)->pRuleset : NULL;
+}
+
+/* get the ruleset of a specifc element of the batch (index not verified!) */
+static inline ruleset_t*
+batchElemGetRuleset(batch_t *pBatch, int i) {
+ return ((msg_t*) pBatch->pElem[i].pUsrp)->pRuleset;
+}
+
+/* get number of msgs for this batch */
+static inline int
+batchNumMsgs(batch_t *pBatch) {
+ return pBatch->nElem;
+}
+
+
+/* set the status of the i-th batch element. Note that once the status is
+ * DISC, it will never be reset. So this function can NOT be used to initialize
+ * the state table. -- rgerhards, 2010-06-10
+ */
+static inline void
+batchSetElemState(batch_t *pBatch, int i, batch_state_t newState) {
+ if(pBatch->pElem[i].state != BATCH_STATE_DISC)
+ pBatch->pElem[i].state = newState;
+}
+
+
+/* check if an element is a valid entry. We do NOT verify if the
+ * element index is valid. -- rgerhards, 2010-06-10
+ */
+static inline int
+batchIsValidElem(batch_t *pBatch, int i) {
+ return(pBatch->pElem[i].bFilterOK && pBatch->pElem[i].state != BATCH_STATE_DISC);
+}
+
+
+/* copy one batch element to another.
+ * This creates a complete duplicate in those cases where
+ * it is needed. Use duplication only when absolutely necessary!
+ * Note that all working fields are reset to zeros. If that were
+ * not done, we would have potential problems with invalid
+ * or double pointer frees.
+ * rgerhards, 2010-06-10
+ */
+static inline void
+batchCopyElem(batch_obj_t *pDest, batch_obj_t *pSrc) {
+ memset(pDest, 0, sizeof(batch_obj_t));
+ pDest->pUsrp = pSrc->pUsrp;
+ pDest->state = pSrc->state;
+}
+
+
+/* free members of a batch "object". Note that we can not do the usual
+ * destruction as the object typically is allocated on the stack and so the
+ * object itself cannot be freed! -- rgerhards, 2010-06-15
+ */
+static inline void
+batchFree(batch_t *pBatch) {
+ int i;
+ int j;
+ for(i = 0 ; i < pBatch->maxElem ; ++i) {
+ for(j = 0 ; j < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++j) {
+ /* staticActParams MUST be freed immediately (if required),
+ * so we do not need to do that!
+ */
+ free(pBatch->pElem[i].staticActStrings[j]);
+ }
+ }
+ free(pBatch->pElem);
+}
+
+
+/* initialiaze a batch "object". The record must already exist,
+ * we "just" initialize it. The max number of elements must be
+ * provided. -- rgerhards, 2010-06-15
+ */
+static inline rsRetVal
+batchInit(batch_t *pBatch, int maxElem) {
+ DEFiRet;
+ pBatch->iDoneUpTo = 0;
+ pBatch->maxElem = maxElem;
+ CHKmalloc(pBatch->pElem = calloc((size_t)maxElem, sizeof(batch_obj_t)));
+ // TODO: replace calloc by inidividual writes?
+finalize_it:
+ RETiRet;
+}
+
+
+/* primarily a helper for debug purposes, get human-readble name of state */
+static inline char *
+batchState2String(batch_state_t state) {
+ switch(state) {
+ case BATCH_STATE_RDY:
+ return "BATCH_STATE_RDY";
+ case BATCH_STATE_BAD:
+ return "BATCH_STATE_BAD";
+ case BATCH_STATE_SUB:
+ return "BATCH_STATE_SUB";
+ case BATCH_STATE_COMM:
+ return "BATCH_STATE_COMM";
+ case BATCH_STATE_DISC:
+ return "BATCH_STATE_DISC";
+ }
+ return "ERROR, batch state not known!";
+}
+#endif /* #ifndef BATCH_H_INCLUDED */
diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c
index 037e9f84..646ab39b 100644
--- a/runtime/cfsysline.c
+++ b/runtime/cfsysline.c
@@ -41,6 +41,7 @@
#include "obj.h"
#include "errmsg.h"
#include "srUtils.h"
+#include "unicode-helper.h"
/* static data */
@@ -511,6 +512,8 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void
CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0));
pStrB = NULL;
+ DBGPRINTF("doGetWord: get newval '%s' (len %d), hdlr %p\n",
+ pNewVal, (int) ustrlen(pNewVal), pSetHdlr);
/* we got the word, now set it */
if(pSetHdlr == NULL) {
/* we should set value directly to var */
diff --git a/runtime/conf.c b/runtime/conf.c
index bbd2147a..8e885a1d 100644
--- a/runtime/conf.c
+++ b/runtime/conf.c
@@ -93,19 +93,18 @@ DEFobjCurrIf(net)
DEFobjCurrIf(rule)
DEFobjCurrIf(ruleset)
-static int iNbrActions; /* number of actions the running config has. Needs to be init on ReInitConf() */
+static int iNbrActions = 0; /* number of currently defined actions */
-/* The following global variables are used for building
+/* The following module-global variables are used for building
* tag and host selector lines during startup and config reload.
* This is stored as a global variable pool because of its ease. It is
* also fairly compatible with multi-threading as the stratup code must
- * be run in a single thread anyways. So there can be no race conditions. These
- * variables are no longer used once the configuration has been loaded (except,
- * of course, during a reload). rgerhards 2005-10-18
+ * be run in a single thread anyways. So there can be no race conditions.
+ * rgerhards 2005-10-18
*/
-EHostnameCmpMode eDfltHostnameCmpMode;
-cstr_t *pDfltHostnameCmp;
-cstr_t *pDfltProgNameCmp;
+static EHostnameCmpMode eDfltHostnameCmpMode = HN_NO_COMP;
+static cstr_t *pDfltHostnameCmp = NULL;
+static cstr_t *pDfltProgNameCmp = NULL;
/* process a directory and include all of its files into
@@ -925,7 +924,7 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f)
}
/* skip to action part */
- if((iRet = parsSkipWhitespace(pPars)) != RS_RET_OK) {
+ if((iRet = parsSkipWhitespace(pPars, 1)) != RS_RET_OK) {
errmsg.LogError(0, iRet, "error %d skipping to action part - ignoring selector", iRet);
rsParsDestruct(pPars);
return(iRet);
@@ -1041,7 +1040,7 @@ static rsRetVal cflineDoFilter(uchar **pp, rule_t *f)
DEFiRet;
ASSERT(pp != NULL);
- ASSERT(f != NULL);
+ ISOBJ_TYPE_assert(f, rule);
/* check which filter we need to pull... */
switch(**pp) {
@@ -1063,6 +1062,7 @@ static rsRetVal cflineDoFilter(uchar **pp, rule_t *f)
* and, if so, we copy them over. rgerhards, 2005-10-18
*/
if(pDfltProgNameCmp != NULL) {
+RUNLOG_STR("dflt ProgNameCmp != NULL, setting opCSProgNameComp");
CHKiRet(rsCStrConstructFromCStr(&(f->pCSProgNameComp), pDfltProgNameCmp));
}
@@ -1092,6 +1092,11 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction)
/* loop through all modules and see if one picks up the line */
pMod = module.GetNxtType(NULL, eMOD_OUT);
+ /* Note: clang static analyzer reports that pMod mybe == NULL. However, this is
+ * not possible, because we have the built-in output modules which are always
+ * present. Anyhow, we guard this by an assert. -- rgerhards, 2010-12-16
+ */
+ assert(pMod != NULL);
while(pMod != NULL) {
pOMSR = NULL;
iRet = pMod->mod.om.parseSelectorAct(p, &pModData, &pOMSR);
@@ -1105,7 +1110,7 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction)
dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n");
pAction->f_ReduceRepeated = 0;
}
- pAction->bEnabled = 1; /* action is enabled */
+ pAction->eState = ACT_STATE_RDY; /* action is enabled */
iNbrActions++; /* one more active action! */
}
break;
@@ -1207,21 +1212,6 @@ cfline(uchar *line, rule_t **pfCurr)
}
-/* Reinitialize the configuration subsystem. This is a "work-around" to the fact
- * that we do not yet have actual config objects. This method is to be called
- * whenever a totally new config is started (which means on startup and HUP).
- * Note that it MUST NOT be called for an included config file.
- * rgerhards, 2008-07-28
- */
-static rsRetVal
-ReInitConf(void)
-{
- DEFiRet;
- iNbrActions = 0; /* this is what we created the function for ;) - action count is reset */
- RETiRet;
-}
-
-
/* return the current number of active actions
* rgerhards, 2008-07-28
*/
@@ -1255,7 +1245,6 @@ CODESTARTobjQueryInterface(conf)
pIf->doIncludeLine = doIncludeLine;
pIf->cfline = cfline;
pIf->processConfFile = processConfFile;
- pIf->ReInitConf = ReInitConf;
pIf->GetNbrActActions = GetNbrActActions;
finalize_it:
@@ -1267,6 +1256,15 @@ ENDobjQueryInterface(conf)
*/
BEGINObjClassExit(conf, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */
CODESTARTObjClassExit(conf)
+ /* free no-longer needed module-global variables */
+ if(pDfltHostnameCmp != NULL) {
+ rsCStrDestruct(&pDfltHostnameCmp);
+ }
+
+ if(pDfltProgNameCmp != NULL) {
+ rsCStrDestruct(&pDfltProgNameCmp);
+ }
+
/* release objects we no longer need */
objRelease(expr, CORE_COMPONENT);
objRelease(ctok, CORE_COMPONENT);
diff --git a/runtime/conf.h b/runtime/conf.h
index 25b887be..d85d1f82 100644
--- a/runtime/conf.h
+++ b/runtime/conf.h
@@ -37,20 +37,18 @@ BEGINinterface(conf) /* name must also be changed in ENDinterface macro! */
rsRetVal (*doIncludeLine)(uchar **pp, __attribute__((unused)) void* pVal);
rsRetVal (*cfline)(uchar *line, rule_t **pfCurr);
rsRetVal (*processConfFile)(uchar *pConfFile);
- rsRetVal (*ReInitConf)(void);
rsRetVal (*GetNbrActActions)(int *);
ENDinterface(conf)
-#define confCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */
+#define confCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */
+/* in Version 3, entry point "ReInitConf()" was removed, as we do not longer need
+ * to support restart-type HUP -- rgerhards, 2009-07-15
+ */
/* prototypes */
PROTOTYPEObj(conf);
-/* TODO: remove them below (means move the config init code) -- rgerhards, 2008-02-19 */
-extern EHostnameCmpMode eDfltHostnameCmpMode;
-extern cstr_t *pDfltHostnameCmp;
-extern cstr_t *pDfltProgNameCmp;
/* TODO: the following 2 need to go in conf obj interface... */
rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName);
rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl);
diff --git a/runtime/datetime.c b/runtime/datetime.c
index 593c3d5c..f81180ea 100644
--- a/runtime/datetime.c
+++ b/runtime/datetime.c
@@ -127,6 +127,24 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds)
}
+/* A fast alternative to getCurrTime() and time() that only obtains
+ * a timestamp like time() does. I was told that gettimeofday(), at
+ * least under Linux, is much faster than time() and I could confirm
+ * this testing. So I created that function as a replacement.
+ * rgerhards, 2009-11-12
+ */
+static time_t
+getTime(time_t *ttSeconds)
+{
+ struct timeval tp;
+
+ if(gettimeofday(&tp, NULL) == -1)
+ return -1;
+
+ if(ttSeconds != NULL)
+ *ttSeconds = tp.tv_sec;
+ return tp.tv_sec;
+}
/*******************************************************************
@@ -842,6 +860,7 @@ CODESTARTobjQueryInterface(datetime)
* of course, also affects the "if" above).
*/
pIf->getCurrTime = getCurrTime;
+ pIf->GetTime = getTime;
pIf->ParseTIMESTAMP3339 = ParseTIMESTAMP3339;
pIf->ParseTIMESTAMP3164 = ParseTIMESTAMP3164;
pIf->formatTimestampToMySQL = formatTimestampToMySQL;
diff --git a/runtime/datetime.h b/runtime/datetime.h
index 9dcce3c5..70bbf416 100644
--- a/runtime/datetime.h
+++ b/runtime/datetime.h
@@ -28,6 +28,7 @@
/* the datetime object */
typedef struct datetime_s {
+ char dummy;
} datetime_t;
@@ -41,8 +42,10 @@ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */
int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf);
int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf, int);
int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf);
+ /* v3, 2009-11-12 */
+ time_t (*GetTime)(time_t *ttSeconds);
ENDinterface(datetime)
-#define datetimeCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */
+#define datetimeCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */
/* interface changes:
* 1 - initial version
* 2 - not compatible to 1 - bugfix required ParseTIMESTAMP3164 to accept char ** as
@@ -50,6 +53,7 @@ ENDinterface(datetime)
* third-party module should call. -- rgerhards, 2008.-09-12
* 3 - taken by v5 branch!
* 4 - formatTimestamp3164 takes a third int parameter
+ * 5 - merge of versions 3 + 4 (2010-03-09)
*/
/* prototypes */
diff --git a/runtime/debug.c b/runtime/debug.c
index 81b45d41..3f1c23bd 100644
--- a/runtime/debug.c
+++ b/runtime/debug.c
@@ -46,6 +46,9 @@
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
+#if _POSIX_TIMERS <= 0
+#include <sys/time.h>
+#endif
#include "rsyslog.h"
#include "debug.h"
@@ -154,9 +157,7 @@ static pthread_key_t keyCallStack;
*/
static void dbgMutexCancelCleanupHdlr(void *pmut)
{
- int ret;
- ret = pthread_mutex_unlock((pthread_mutex_t*) pmut);
- assert(ret == 0);
+ pthread_mutex_unlock((pthread_mutex_t*) pmut);
}
@@ -470,7 +471,7 @@ static inline void dbgMutexLockLog(pthread_mutex_t *pmut, dbgFuncDB_t *pFuncDB,
dbgMutLogDelEntry(pLog);
/* add "lock" entry */
- pLog = dbgMutLogAddEntry(pmut, MUTOP_LOCK, pFuncDB, lockLn);
+ dbgMutLogAddEntry(pmut, MUTOP_LOCK, pFuncDB, lockLn);
dbgFuncDBAddMutexLock(pFuncDB, pmut, lockLn);
pthread_mutex_unlock(&mutMutLog);
if(bPrintMutexAction)
@@ -517,7 +518,7 @@ static inline void dbgMutexTryLockLog(pthread_mutex_t *pmut, dbgFuncDB_t *pFuncD
dbgMutLogDelEntry(pLog);
/* add "lock" entry */
- pLog = dbgMutLogAddEntry(pmut, MUTOP_LOCK, pFuncDB, lockLn);
+ dbgMutLogAddEntry(pmut, MUTOP_LOCK, pFuncDB, lockLn);
dbgFuncDBAddMutexLock(pFuncDB, pmut, lockLn);
pthread_mutex_unlock(&mutMutLog);
if(bPrintMutexAction)
@@ -831,13 +832,12 @@ sigsegvHdlr(int signum)
abort();
}
-#if 1
-#pragma GCC diagnostic ignored "-Wempty-body"
-/* write the debug message. This is a helper to dbgprintf and dbgoprint which
- * contains common code. added 2008-09-26 rgerhards
+/* actually write the debug message. This is a separate fuction because the cleanup_push/_pop
+ * interface otherwise is unsafe to use (generates compiler warnings at least).
+ * 2009-05-20 rgerhards
*/
-static void
-dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg)
+static inline void
+do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg)
{
static pthread_t ptLastThrdID = 0;
static int bWasNL = 0;
@@ -845,22 +845,9 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg)
char pszWriteBuf[32*1024];
size_t lenWriteBuf;
struct timespec t;
- uchar *pszObjName = NULL;
- int ret;
-
- /* we must get the object name before we lock the mutex, because the object
- * potentially calls back into us. If we locked the mutex, we would deadlock
- * ourselfs. On the other hand, the GetName call needs not to be protected, as
- * this thread has a valid reference. If such an object is deleted by another
- * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26
- */
- if(pObj != NULL) {
- pszObjName = obj.GetName(pObj);
- }
-
- ret = pthread_mutex_lock(&mutdbgprint);
- assert(ret == 0); /* make sure mutex operation does not fail */
- pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint);
+# if _POSIX_TIMERS <= 0
+ struct timeval tv;
+# endif
/* The bWasNL handler does not really work. It works if no thread
* switching occurs during non-NL messages. Else, things are messed
@@ -886,7 +873,14 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg)
if(bWasNL) {
if(bPrintTime) {
+# if _POSIX_TIMERS > 0
+ /* this is the "regular" code */
clock_gettime(CLOCK_REALTIME, &t);
+# else
+ gettimeofday(&tv, NULL);
+ t.tv_sec = tv.tv_sec;
+ t.tv_nsec = tv.tv_usec * 1000;
+# endif
lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf),
"%4.4ld.%9.9ld:", (long) (t.tv_sec % 10000), t.tv_nsec);
if(stddbg != -1) write(stddbg, pszWriteBuf, lenWriteBuf);
@@ -908,11 +902,35 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg)
if(altdbg != -1) write(altdbg, pszMsg, lenMsg);
bWasNL = (pszMsg[lenMsg - 1] == '\n') ? 1 : 0;
+}
+
+#pragma GCC diagnostic ignored "-Wempty-body"
+/* write the debug message. This is a helper to dbgprintf and dbgoprint which
+ * contains common code. added 2008-09-26 rgerhards
+ */
+static void
+dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg)
+{
+ uchar *pszObjName = NULL;
+
+ /* we must get the object name before we lock the mutex, because the object
+ * potentially calls back into us. If we locked the mutex, we would deadlock
+ * ourselfs. On the other hand, the GetName call needs not to be protected, as
+ * this thread has a valid reference. If such an object is deleted by another
+ * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26
+ */
+ if(pObj != NULL) {
+ pszObjName = obj.GetName(pObj);
+ }
+
+ pthread_mutex_lock(&mutdbgprint);
+ pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint);
+
+ do_dbgprint(pszObjName, pszMsg, lenMsg);
pthread_cleanup_pop(1);
}
#pragma GCC diagnostic warning "-Wempty-body"
-#endif
/* print some debug output when an object is given
* This is mostly a copy of dbgprintf, but I do not know how to combine it
@@ -1073,7 +1091,9 @@ int dbgEntrFunc(dbgFuncDB_t **ppFuncDB, const char *file, const char *func, int
/* when we reach this point, we have a fully-initialized FuncDB! */
PREFER_ATOMIC_INC(pFuncDB->nTimesCalled);
if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot))
- dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func);
+ if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */
+ dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func);
+ }
if(pThrd->stackPtr >= (int) (sizeof(pThrd->callStack) / sizeof(dbgFuncDB_t*))) {
dbgprintf("%s:%d: %s: debug module: call stack for this thread full, suspending call tracking\n",
pFuncDB->file, pFuncDB->line, pFuncDB->func);
@@ -1103,10 +1123,12 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet)
dbgFuncDBPrintActiveMutexes(pFuncDB, "WARNING: mutex still owned by us as we exit function, mutex: ", pthread_self());
if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) {
- if(iRet == RS_RET_NO_IRET)
- dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func);
- else
- dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet);
+ if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */
+ if(iRet == RS_RET_NO_IRET)
+ dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func);
+ else
+ dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet);
+ }
}
pThrd->stackPtr = iStackPtrRestore;
if(pThrd->stackPtr < 0) {
@@ -1252,6 +1274,20 @@ dbgPrintNameIsInList(const uchar *pName, dbgPrintName_t *pRoot)
}
+/* this is a special version of malloc that fills the alloced memory with
+ * HIGHVALUE, as this helps to identify bugs. -- rgerhards, 2009-10-22
+ */
+void *
+dbgmalloc(size_t size)
+{
+ void *pRet;
+ pRet = malloc(size);
+ if(pRet != NULL)
+ memset(pRet, 0xff, size);
+ return pRet;
+}
+
+
/* read in the runtime options
* rgerhards, 2008-02-28
*/
@@ -1270,7 +1306,7 @@ dbgGetRuntimeOptions(void)
while(dbgGetRTOptNamVal(&pszOpts, &optname, &optval)) {
if(!strcasecmp((char*)optname, "help")) {
fprintf(stderr,
- "rsyslogd runtime debug support - help requested, rsyslog terminates\n\n"
+ "rsyslogd " VERSION " runtime debug support - help requested, rsyslog terminates\n\n"
"environment variables:\n"
"addional logfile: export RSYSLOG_DEBUGFILE=\"/path/to/file\"\n"
"to set: export RSYSLOG_DEBUG=\"cmd cmd cmd\"\n\n"
@@ -1318,7 +1354,7 @@ dbgGetRuntimeOptions(void)
bAbortTrace = 0;
} else if(!strcasecmp((char*)optname, "filetrace")) {
if(*optval == '\0') {
- fprintf(stderr, "Error: logfile debug option requires filename, "
+ fprintf(stderr, "rsyslogd " VERSION " error: logfile debug option requires filename, "
"e.g. \"logfile=debug.c\"\n");
exit(1);
} else {
@@ -1326,7 +1362,7 @@ dbgGetRuntimeOptions(void)
dbgPrintNameAdd(optval, &printNameFileRoot);
}
} else {
- fprintf(stderr, "Error: invalid debug option '%s', value '%s' - ignored\n",
+ fprintf(stderr, "rsyslogd " VERSION " error: invalid debug option '%s', value '%s' - ignored\n",
optval, optname);
}
}
diff --git a/runtime/debug.h b/runtime/debug.h
index cfdf819c..c011dd2d 100644
--- a/runtime/debug.h
+++ b/runtime/debug.h
@@ -3,7 +3,7 @@
* Definitions for the debug and run-time analysis support module.
* Contains a lot of macros.
*
- * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -104,6 +104,7 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet);
void dbgSetExecLocation(int iStackPtr, int line);
void dbgSetThrdName(uchar *pszName);
void dbgPrintAllDebugInfo(void);
+void *dbgmalloc(size_t size);
/* macros */
#define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); }
@@ -131,6 +132,12 @@ void dbgPrintAllDebugInfo(void);
# define RUNLOG_STR(str)
#endif
+#ifdef MEMCHECK
+# define MALLOC(x) dbgmalloc(x)
+#else
+# define MALLOC(x) malloc(x)
+#endif
+
/* mutex operations */
#define MUTOP_LOCKWAIT 1
#define MUTOP_LOCK 2
diff --git a/runtime/errmsg.h b/runtime/errmsg.h
index 799954fb..ac7018b3 100644
--- a/runtime/errmsg.h
+++ b/runtime/errmsg.h
@@ -30,6 +30,7 @@
/* the errmsg object */
typedef struct errmsg_s {
+ char dummy;
} errmsg_t;
diff --git a/runtime/expr.h b/runtime/expr.h
index 974b71ec..1afe1a1f 100644
--- a/runtime/expr.h
+++ b/runtime/expr.h
@@ -30,6 +30,7 @@
/* a node inside an expression tree */
typedef struct exprNode_s {
+ char dummy;
} exprNode_t;
diff --git a/runtime/glbl.c b/runtime/glbl.c
index 59d1fb0f..dea5a17b 100644
--- a/runtime/glbl.c
+++ b/runtime/glbl.c
@@ -31,6 +31,9 @@
#include "config.h"
#include <stdlib.h>
#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
#include <assert.h>
#include "rsyslog.h"
@@ -39,6 +42,8 @@
#include "cfsysline.h"
#include "glbl.h"
#include "prop.h"
+#include "atomic.h"
+#include "errmsg.h"
/* some defaults */
#ifndef DFLT_NETSTRM_DRVR
@@ -48,6 +53,7 @@
/* static data */
DEFobjStaticHelpers
DEFobjCurrIf(prop)
+DEFobjCurrIf(errmsg)
/* static data
* For this object, these variables are obviously what makes the "meat" of the
@@ -55,7 +61,7 @@ DEFobjCurrIf(prop)
*/
static uchar *pszWorkDir = NULL;
static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */
-static int bHUPisRestart = 0; /* should SIGHUP cause a full system restart? */
+static int bParseHOSTNAMEandTAG = 1; /* parser modification (based on startup params!) */
static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */
static int iMaxLine = 2048; /* maximum length of a syslog message */
static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */
@@ -73,6 +79,10 @@ static uchar *pszDfltNetstrmDrvr = NULL; /* module name of default netstream dri
static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm driver */
static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */
static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */
+static int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */
+#ifndef HAVE_ATOMIC_BUILTINS
+static DEF_ATOMIC_HELPER_MUT(mutTerminateInputs);
+#endif
#ifdef USE_UNLIMITED_SELECT
static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* size of select() bitmask in bytes */
#endif
@@ -97,9 +107,9 @@ static dataType Get##nameFunc(void) \
return(nameVar); \
}
+SIMP_PROP(ParseHOSTNAMEandTAG, bParseHOSTNAMEandTAG, int)
SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int)
SIMP_PROP(PreserveFQDN, bPreserveFQDN, int)
-SIMP_PROP(HUPisRestart, bHUPisRestart, int)
SIMP_PROP(MaxLine, iMaxLine, int)
SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */
SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int)
@@ -124,6 +134,74 @@ SIMP_PROP_SET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) /* TO
#undef SIMP_PROP_GET
+/* return global input termination status
+ * rgerhards, 2009-07-20
+ */
+static int GetGlobalInputTermState(void)
+{
+ return ATOMIC_FETCH_32BIT(&bTerminateInputs, &mutTerminateInputs);
+}
+
+
+/* set global termiantion state to "terminate". Note that this is a
+ * "once in a lifetime" action which can not be undone. -- gerhards, 2009-07-20
+ */
+static void SetGlobalInputTermination(void)
+{
+ ATOMIC_STORE_1_TO_INT(&bTerminateInputs, &mutTerminateInputs);
+}
+
+
+/* This function is used to set the global work directory name.
+ * It verifies that the provided directory actually exists and
+ * emits an error message if not.
+ * rgerhards, 2011-02-16
+ */
+static rsRetVal setWorkDir(void __attribute__((unused)) *pVal, uchar *pNewVal)
+{
+ size_t lenDir;
+ int i;
+ struct stat sb;
+ DEFiRet;
+
+ /* remove trailing slashes */
+ lenDir = ustrlen(pNewVal);
+ i = lenDir - 1;
+ while(i > 0 && pNewVal[i] == '/') {
+ --i;
+ }
+
+ if(i < 0) {
+ errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: empty value "
+ "- directive ignored");
+ ABORT_FINALIZE(RS_RET_ERR_WRKDIR);
+ }
+
+ if(i != (int) lenDir - 1) {
+ pNewVal[i+1] = '\0';
+ errmsg.LogError(0, RS_RET_WRN_WRKDIR, "$WorkDirectory: trailing slashes "
+ "removed, new value is '%s'", pNewVal);
+ }
+
+ if(stat((char*) pNewVal, &sb) != 0) {
+ errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: %s can not be "
+ "accessed, probably does not exist - directive ignored", pNewVal);
+ ABORT_FINALIZE(RS_RET_ERR_WRKDIR);
+ }
+
+ if(!S_ISDIR(sb.st_mode)) {
+ errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: %s not a directory - directive ignored",
+ pNewVal);
+ ABORT_FINALIZE(RS_RET_ERR_WRKDIR);
+ }
+
+ free(pszWorkDir);
+ pszWorkDir = pNewVal;
+
+finalize_it:
+ RETiRet;
+}
+
/* return our local hostname. if it is not set, "[localhost]" is returned
*/
static uchar*
@@ -131,6 +209,11 @@ GetLocalHostName(void)
{
uchar *pszRet;
+ if(LocalHostNameOverride != NULL) {
+ pszRet = LocalHostNameOverride;
+ goto done;
+ }
+
if(LocalHostName == NULL)
pszRet = (uchar*) "[localhost]";
else {
@@ -139,6 +222,7 @@ GetLocalHostName(void)
else
pszRet = LocalHostName;
}
+done:
return(pszRet);
}
@@ -177,6 +261,7 @@ finalize_it:
RETiRet;
}
+
/* return our local hostname as a string property
*/
static prop_t*
@@ -253,13 +338,15 @@ CODESTARTobjQueryInterface(glbl)
pIf->GetWorkDir = GetWorkDir;
pIf->GenerateLocalHostNameProperty = GenerateLocalHostNameProperty;
pIf->GetLocalHostNameProp = GetLocalHostNameProp;
+ pIf->SetGlobalInputTermination = SetGlobalInputTermination;
+ pIf->GetGlobalInputTermState = GetGlobalInputTermState;
#define SIMP_PROP(name) \
pIf->Get##name = Get##name; \
pIf->Set##name = Set##name;
SIMP_PROP(MaxLine);
SIMP_PROP(OptimizeUniProc);
+ SIMP_PROP(ParseHOSTNAMEandTAG);
SIMP_PROP(PreserveFQDN);
- SIMP_PROP(HUPisRestart);
SIMP_PROP(DefPFFamily);
SIMP_PROP(DropMalPTRMsgs);
SIMP_PROP(Option_DisallowWarning);
@@ -312,7 +399,6 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
}
bDropMalPTRMsgs = 0;
bOptimizeUniProc = 1;
- bHUPisRestart = 0;
bPreserveFQDN = 0;
#ifdef USE_UNLIMITED_SELECT
iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask);
@@ -329,9 +415,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */
/* request objects we use */
CHKiRet(objUse(prop, CORE_COMPONENT));
+ CHKiRet(objUse(errmsg, CORE_COMPONENT));
/* register config handlers (TODO: we need to implement a way to unregister them) */
- CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, NULL, &pszWorkDir, NULL));
+ CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, setWorkDir, NULL, NULL));
CHKiRet(regCfSysLineHdlr((uchar *)"dropmsgswithmaliciousdnsptrrecords", 0, eCmdHdlrBinary, NULL, &bDropMalPTRMsgs, NULL));
CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvr, NULL));
CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL));
@@ -339,9 +426,10 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */
CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL));
CHKiRet(regCfSysLineHdlr((uchar *)"localhostname", 0, eCmdHdlrGetWord, NULL, &LocalHostNameOverride, NULL));
CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL));
- CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL));
CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL));
CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL));
+
+ INIT_ATOMIC_HELPER_MUT(mutTerminateInputs);
ENDObjClassInit(glbl)
@@ -365,6 +453,7 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */
if(LocalFQDNName != NULL)
free(LocalFQDNName);
objRelease(prop, CORE_COMPONENT);
+ DESTROY_ATOMIC_HELPER_MUT(mutTerminateInputs);
ENDObjClassExit(glbl)
/* vi:set ai:
diff --git a/runtime/glbl.h b/runtime/glbl.h
index 6a332576..4b4bdf83 100644
--- a/runtime/glbl.h
+++ b/runtime/glbl.h
@@ -8,7 +8,7 @@
* Please note that there currently is no glbl.c file as we do not yet
* have any implementations.
*
- * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -44,7 +44,6 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */
rsRetVal (*Set##name)(dataType);
SIMP_PROP(MaxLine, int)
SIMP_PROP(OptimizeUniProc, int)
- SIMP_PROP(HUPisRestart, int)
SIMP_PROP(PreserveFQDN, int)
SIMP_PROP(DefPFFamily, int)
SIMP_PROP(DropMalPTRMsgs, int)
@@ -62,6 +61,11 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */
/* added v3, 2009-06-30 */
rsRetVal (*GenerateLocalHostNameProperty)(void);
prop_t* (*GetLocalHostNameProp)(void);
+ /* added v4, 2009-07-20 */
+ int (*GetGlobalInputTermState)(void);
+ void (*SetGlobalInputTermination)(void);
+ /* added v5, 2009-11-03 */
+ SIMP_PROP(ParseHOSTNAMEandTAG, int)
/* note: v4, v5 are already used by more recent versions, so we need to skip them! */
/* added v6, 2009-11-16 as part of varmojfekoj's "unlimited select()" patch
* Note that it must be always present, otherwise the interface would have different
@@ -71,9 +75,11 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */
* at some later stage).
*/
SIMP_PROP(FdSetSize, int)
+ /* v7: was neeeded to mean v5+v6 - do NOT add anything else for that version! */
+ /* next change is v8! */
#undef SIMP_PROP
ENDinterface(glbl)
-#define glblCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */
+#define glblCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */
/* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */
/* the remaining prototypes */
diff --git a/runtime/hashtable.c b/runtime/hashtable.c
new file mode 100644
index 00000000..a01fa7d9
--- /dev/null
+++ b/runtime/hashtable.c
@@ -0,0 +1,323 @@
+/* Copyright (C) 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */
+
+#include "hashtable.h"
+#include "hashtable_private.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+
+/*
+Credit for primes table: Aaron Krowne
+ http://br.endernet.org/~akrowne/
+ http://planetmath.org/encyclopedia/GoodHashTablePrimes.html
+*/
+static const unsigned int primes[] = {
+53, 97, 193, 389,
+769, 1543, 3079, 6151,
+12289, 24593, 49157, 98317,
+196613, 393241, 786433, 1572869,
+3145739, 6291469, 12582917, 25165843,
+50331653, 100663319, 201326611, 402653189,
+805306457, 1610612741
+};
+const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]);
+
+#define MAX_LOAD_FACTOR 65 /* to get real factor, divide by 100! */
+
+/* compute max load. We use a constant factor of 0.65, but do
+ * everything times 100, so that we do not need floats.
+ */
+static inline unsigned
+getLoadLimit(unsigned size)
+{
+ return (unsigned int) ((unsigned long long) size * MAX_LOAD_FACTOR) / 100;
+}
+
+/*****************************************************************************/
+struct hashtable *
+create_hashtable(unsigned int minsize,
+ unsigned int (*hashf) (void*),
+ int (*eqf) (void*,void*), void (*dest)(void*))
+{
+ struct hashtable *h;
+ unsigned int pindex, size = primes[0];
+ /* Check requested hashtable isn't too large */
+ if (minsize > (1u << 30)) return NULL;
+ /* Enforce size as prime */
+ for (pindex=0; pindex < prime_table_length; pindex++) {
+ if (primes[pindex] > minsize) { size = primes[pindex]; break; }
+ }
+ h = (struct hashtable *)malloc(sizeof(struct hashtable));
+ if (NULL == h) return NULL; /*oom*/
+ h->table = (struct entry **)malloc(sizeof(struct entry*) * size);
+ if (NULL == h->table) { free(h); return NULL; } /*oom*/
+ memset(h->table, 0, size * sizeof(struct entry *));
+ h->tablelength = size;
+ h->primeindex = pindex;
+ h->entrycount = 0;
+ h->hashfn = hashf;
+ h->eqfn = eqf;
+ h->dest = dest;
+ h->loadlimit = getLoadLimit(size);
+ return h;
+}
+
+/*****************************************************************************/
+unsigned int
+hash(struct hashtable *h, void *k)
+{
+ /* Aim to protect against poor hash functions by adding logic here
+ * - logic taken from java 1.4 hashtable source */
+ unsigned int i = h->hashfn(k);
+ i += ~(i << 9);
+ i ^= ((i >> 14) | (i << 18)); /* >>> */
+ i += (i << 4);
+ i ^= ((i >> 10) | (i << 22)); /* >>> */
+ return i;
+}
+
+/*****************************************************************************/
+static int
+hashtable_expand(struct hashtable *h)
+{
+ /* Double the size of the table to accomodate more entries */
+ struct entry **newtable;
+ struct entry *e;
+ struct entry **pE;
+ unsigned int newsize, i, idx;
+ /* Check we're not hitting max capacity */
+ if (h->primeindex == (prime_table_length - 1)) return 0;
+ newsize = primes[++(h->primeindex)];
+
+ newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize);
+ if (NULL != newtable)
+ {
+ memset(newtable, 0, newsize * sizeof(struct entry *));
+ /* This algorithm is not 'stable'. ie. it reverses the list
+ * when it transfers entries between the tables */
+ for (i = 0; i < h->tablelength; i++) {
+ while (NULL != (e = h->table[i])) {
+ h->table[i] = e->next;
+ idx = indexFor(newsize,e->h);
+ e->next = newtable[idx];
+ newtable[idx] = e;
+ }
+ }
+ free(h->table);
+ h->table = newtable;
+ }
+ /* Plan B: realloc instead */
+ else
+ {
+ newtable = (struct entry **)
+ realloc(h->table, newsize * sizeof(struct entry *));
+ if (NULL == newtable) { (h->primeindex)--; return 0; }
+ h->table = newtable;
+ memset(newtable[h->tablelength], 0, newsize - h->tablelength);
+ for (i = 0; i < h->tablelength; i++) {
+ for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) {
+ idx = indexFor(newsize,e->h);
+ if (idx == i)
+ {
+ pE = &(e->next);
+ }
+ else
+ {
+ *pE = e->next;
+ e->next = newtable[idx];
+ newtable[idx] = e;
+ }
+ }
+ }
+ }
+ h->tablelength = newsize;
+ h->loadlimit = getLoadLimit(newsize);
+ return -1;
+}
+
+/*****************************************************************************/
+unsigned int
+hashtable_count(struct hashtable *h)
+{
+ return h->entrycount;
+}
+
+/*****************************************************************************/
+int
+hashtable_insert(struct hashtable *h, void *k, void *v)
+{
+ /* This method allows duplicate keys - but they shouldn't be used */
+ unsigned int idx;
+ struct entry *e;
+ if (++(h->entrycount) > h->loadlimit)
+ {
+ /* Ignore the return value. If expand fails, we should
+ * still try cramming just this value into the existing table
+ * -- we may not have memory for a larger table, but one more
+ * element may be ok. Next time we insert, we'll try expanding again.*/
+ hashtable_expand(h);
+ }
+ e = (struct entry *)malloc(sizeof(struct entry));
+ if (NULL == e) { --(h->entrycount); return 0; } /*oom*/
+ e->h = hash(h,k);
+ idx = indexFor(h->tablelength,e->h);
+ e->k = k;
+ e->v = v;
+ e->next = h->table[idx];
+ h->table[idx] = e;
+ return -1;
+}
+
+/*****************************************************************************/
+void * /* returns value associated with key */
+hashtable_search(struct hashtable *h, void *k)
+{
+ struct entry *e;
+ unsigned int hashvalue, idx;
+ hashvalue = hash(h,k);
+ idx = indexFor(h->tablelength,hashvalue);
+ e = h->table[idx];
+ while (NULL != e)
+ {
+ /* Check hash value to short circuit heavier comparison */
+ if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v;
+ e = e->next;
+ }
+ return NULL;
+}
+
+/*****************************************************************************/
+void * /* returns value associated with key */
+hashtable_remove(struct hashtable *h, void *k)
+{
+ /* TODO: consider compacting the table when the load factor drops enough,
+ * or provide a 'compact' method. */
+
+ struct entry *e;
+ struct entry **pE;
+ void *v;
+ unsigned int hashvalue, idx;
+
+ hashvalue = hash(h,k);
+ idx = indexFor(h->tablelength,hash(h,k));
+ pE = &(h->table[idx]);
+ e = *pE;
+ while (NULL != e)
+ {
+ /* Check hash value to short circuit heavier comparison */
+ if ((hashvalue == e->h) && (h->eqfn(k, e->k)))
+ {
+ *pE = e->next;
+ h->entrycount--;
+ v = e->v;
+ freekey(e->k);
+ free(e);
+ return v;
+ }
+ pE = &(e->next);
+ e = e->next;
+ }
+ return NULL;
+}
+
+/*****************************************************************************/
+/* destroy */
+void
+hashtable_destroy(struct hashtable *h, int free_values)
+{
+ unsigned int i;
+ struct entry *e, *f;
+ struct entry **table = h->table;
+ if (free_values)
+ {
+ for (i = 0; i < h->tablelength; i++)
+ {
+ e = table[i];
+ while (NULL != e)
+ {
+ f = e;
+ e = e->next;
+ freekey(f->k);
+ if(h->dest == NULL)
+ free(f->v);
+ else
+ h->dest(f->v);
+ free(f);
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < h->tablelength; i++)
+ {
+ e = table[i];
+ while (NULL != e)
+ { f = e; e = e->next; freekey(f->k); free(f); }
+ }
+ }
+ free(h->table);
+ free(h);
+}
+
+/* some generic hash functions */
+
+/* one provided by Aaaron Wiebe based on perl's hashng algorithm
+ * (so probably pretty generic). Not for excessively large strings!
+ */
+unsigned int
+hash_from_string(void *k)
+{
+ int len;
+ char *rkey = (char*) k;
+ unsigned hashval = 1;
+
+ len = (int) strlen(rkey);
+ while (len--)
+ hashval = hashval * 33 + *rkey++;
+
+ return hashval;
+}
+
+
+int
+key_equals_string(void *key1, void *key2)
+{
+ /* we must return true IF the keys are equal! */
+ return !strcmp(key1, key2);
+}
+
+
+/*
+ * Copyright (c) 2002, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
diff --git a/runtime/hashtable.h b/runtime/hashtable.h
new file mode 100644
index 00000000..f777ad0b
--- /dev/null
+++ b/runtime/hashtable.h
@@ -0,0 +1,202 @@
+/* Copyright (C) 2002 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+
+#ifndef __HASHTABLE_CWC22_H__
+#define __HASHTABLE_CWC22_H__
+
+struct hashtable;
+
+/* Example of use:
+ *
+ * struct hashtable *h;
+ * struct some_key *k;
+ * struct some_value *v;
+ *
+ * static unsigned int hash_from_key_fn( void *k );
+ * static int keys_equal_fn ( void *key1, void *key2 );
+ *
+ * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn);
+ * k = (struct some_key *) malloc(sizeof(struct some_key));
+ * v = (struct some_value *) malloc(sizeof(struct some_value));
+ *
+ * (initialise k and v to suitable values)
+ *
+ * if (! hashtable_insert(h,k,v) )
+ * { exit(-1); }
+ *
+ * if (NULL == (found = hashtable_search(h,k) ))
+ * { printf("not found!"); }
+ *
+ * if (NULL == (found = hashtable_remove(h,k) ))
+ * { printf("Not found\n"); }
+ *
+ */
+
+/* Macros may be used to define type-safe(r) hashtable access functions, with
+ * methods specialized to take known key and value types as parameters.
+ *
+ * Example:
+ *
+ * Insert this at the start of your file:
+ *
+ * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value);
+ * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value);
+ * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value);
+ *
+ * This defines the functions 'insert_some', 'search_some' and 'remove_some'.
+ * These operate just like hashtable_insert etc., with the same parameters,
+ * but their function signatures have 'struct some_key *' rather than
+ * 'void *', and hence can generate compile time errors if your program is
+ * supplying incorrect data as a key (and similarly for value).
+ *
+ * Note that the hash and key equality functions passed to create_hashtable
+ * still take 'void *' parameters instead of 'some key *'. This shouldn't be
+ * a difficult issue as they're only defined and passed once, and the other
+ * functions will ensure that only valid keys are supplied to them.
+ *
+ * The cost for this checking is increased code size and runtime overhead
+ * - if performance is important, it may be worth switching back to the
+ * unsafe methods once your program has been debugged with the safe methods.
+ * This just requires switching to some simple alternative defines - eg:
+ * #define insert_some hashtable_insert
+ *
+ */
+
+/*****************************************************************************
+ * create_hashtable
+
+ * @name create_hashtable
+ * @param minsize minimum initial size of hashtable
+ * @param hashfunction function for hashing keys
+ * @param key_eq_fn function for determining key equality
+ * @param dest destructor for value entries (NULL -> use free())
+ * @return newly created hashtable or NULL on failure
+ */
+
+struct hashtable *
+create_hashtable(unsigned int minsize,
+ unsigned int (*hashfunction) (void*),
+ int (*key_eq_fn) (void*,void*), void (*dest) (void*));
+
+/*****************************************************************************
+ * hashtable_insert
+
+ * @name hashtable_insert
+ * @param h the hashtable to insert into
+ * @param k the key - hashtable claims ownership and will free on removal
+ * @param v the value - does not claim ownership
+ * @return non-zero for successful insertion
+ *
+ * This function will cause the table to expand if the insertion would take
+ * the ratio of entries to table size over the maximum load factor.
+ *
+ * This function does not check for repeated insertions with a duplicate key.
+ * The value returned when using a duplicate key is undefined -- when
+ * the hashtable changes size, the order of retrieval of duplicate key
+ * entries is reversed.
+ * If in doubt, remove before insert.
+ */
+
+int
+hashtable_insert(struct hashtable *h, void *k, void *v);
+
+#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \
+int fnname (struct hashtable *h, keytype *k, valuetype *v) \
+{ \
+ return hashtable_insert(h,k,v); \
+}
+
+/*****************************************************************************
+ * hashtable_search
+
+ * @name hashtable_search
+ * @param h the hashtable to search
+ * @param k the key to search for - does not claim ownership
+ * @return the value associated with the key, or NULL if none found
+ */
+
+void *
+hashtable_search(struct hashtable *h, void *k);
+
+#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \
+valuetype * fnname (struct hashtable *h, keytype *k) \
+{ \
+ return (valuetype *) (hashtable_search(h,k)); \
+}
+
+/*****************************************************************************
+ * hashtable_remove
+
+ * @name hashtable_remove
+ * @param h the hashtable to remove the item from
+ * @param k the key to search for - does not claim ownership
+ * @return the value associated with the key, or NULL if none found
+ */
+
+void * /* returns value */
+hashtable_remove(struct hashtable *h, void *k);
+
+#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \
+valuetype * fnname (struct hashtable *h, keytype *k) \
+{ \
+ return (valuetype *) (hashtable_remove(h,k)); \
+}
+
+
+/*****************************************************************************
+ * hashtable_count
+
+ * @name hashtable_count
+ * @param h the hashtable
+ * @return the number of items stored in the hashtable
+ */
+unsigned int
+hashtable_count(struct hashtable *h);
+
+
+/*****************************************************************************
+ * hashtable_destroy
+
+ * @name hashtable_destroy
+ * @param h the hashtable
+ * @param free_values whether to call 'free' on the remaining values
+ */
+
+void
+hashtable_destroy(struct hashtable *h, int free_values);
+
+#endif /* __HASHTABLE_CWC22_H__ */
+
+/*
+ * Copyright (c) 2002, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+unsigned int hash_from_string(void *k) ;
+int key_equals_string(void *key1, void *key2);
diff --git a/runtime/hashtable/Makefile b/runtime/hashtable/Makefile
new file mode 100644
index 00000000..3b7b5e9f
--- /dev/null
+++ b/runtime/hashtable/Makefile
@@ -0,0 +1,26 @@
+
+tester: hashtable.o tester.o hashtable_itr.o
+ gcc -g -Wall -O -lm -o tester hashtable.o hashtable_itr.o tester.o
+
+all: tester old_tester
+
+tester.o: tester.c
+ gcc -g -Wall -O -c tester.c -o tester.o
+
+old_tester: hashtable_powers.o tester.o hashtable_itr.o
+ gcc -g -Wall -O -o old_tester hashtable_powers.o hashtable_itr.o tester.o
+
+hashtable_powers.o: hashtable_powers.c
+ gcc -g -Wall -O -c hashtable_powers.c -o hashtable_powers.o
+
+hashtable.o: hashtable.c
+ gcc -g -Wall -O -c hashtable.c -o hashtable.o
+
+hashtable_itr.o: hashtable_itr.c
+ gcc -g -Wall -O -c hashtable_itr.c -o hashtable_itr.o
+
+tidy:
+ rm *.o
+
+clean: tidy
+ rm -f tester old_tester
diff --git a/runtime/hashtable/README b/runtime/hashtable/README
new file mode 100644
index 00000000..5cadde0c
--- /dev/null
+++ b/runtime/hashtable/README
@@ -0,0 +1,11 @@
+This is the hashtable code provided by
+Christopher Clark <firstname.lastname@cl.cam.ac.uk>
+available at http://www.cl.cam.ac.uk/~cwc22/hashtable/
+
+It may be slightly modified. The plan is to streamline
+the code based on our needs and "really" integrate it into
+the rsyslog runtime library. For the time being, we use it from
+inside this subdirectory. We do not need all files, but I thought
+I keep them together in case we later need something else.
+
+rgerhards, 2010-09-28
diff --git a/runtime/hashtable/hashtable_utility.c b/runtime/hashtable/hashtable_utility.c
new file mode 100644
index 00000000..c3176709
--- /dev/null
+++ b/runtime/hashtable/hashtable_utility.c
@@ -0,0 +1,71 @@
+/* Copyright (C) 2002 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+
+#include "hashtable.h"
+#include "hashtable_private.h"
+#include "hashtable_utility.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+/*****************************************************************************/
+/* hashtable_change
+ *
+ * function to change the value associated with a key, where there already
+ * exists a value bound to the key in the hashtable.
+ * Source due to Holger Schemel.
+ *
+ * */
+int
+hashtable_change(struct hashtable *h, void *k, void *v)
+{
+ struct entry *e;
+ unsigned int hashvalue, index;
+ hashvalue = hash(h,k);
+ index = indexFor(h->tablelength,hashvalue);
+ e = h->table[index];
+ while (NULL != e)
+ {
+ /* Check hash value to short circuit heavier comparison */
+ if ((hashvalue == e->h) && (h->eqfn(k, e->k)))
+ {
+ free(e->v);
+ e->v = v;
+ return -1;
+ }
+ e = e->next;
+ }
+ return 0;
+}
+
+/*
+ * Copyright (c) 2002, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
diff --git a/runtime/hashtable/hashtable_utility.h b/runtime/hashtable/hashtable_utility.h
new file mode 100644
index 00000000..56a0ffd1
--- /dev/null
+++ b/runtime/hashtable/hashtable_utility.h
@@ -0,0 +1,55 @@
+/* Copyright (C) 2002 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+
+#ifndef __HASHTABLE_CWC22_UTILITY_H__
+#define __HASHTABLE_CWC22_UTILITY_H__
+
+/*****************************************************************************
+ * hashtable_change
+ *
+ * function to change the value associated with a key, where there already
+ * exists a value bound to the key in the hashtable.
+ * Source due to Holger Schemel.
+ *
+ * @name hashtable_change
+ * @param h the hashtable
+ * @param key
+ * @param value
+ *
+ */
+int
+hashtable_change(struct hashtable *h, void *k, void *v);
+
+#endif /* __HASHTABLE_CWC22_H__ */
+
+/*
+ * Copyright (c) 2002, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
diff --git a/runtime/hashtable/tester.c b/runtime/hashtable/tester.c
new file mode 100644
index 00000000..4678ffa8
--- /dev/null
+++ b/runtime/hashtable/tester.c
@@ -0,0 +1,270 @@
+/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+
+#include "hashtable.h"
+#include "hashtable_itr.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h> /* for memcmp */
+
+static const int ITEM_COUNT = 4000;
+
+typedef unsigned int uint32_t;
+typedef unsigned short uint16_t;
+
+/*****************************************************************************/
+struct key
+{
+ uint32_t one_ip; uint32_t two_ip; uint16_t one_port; uint16_t two_port;
+};
+
+struct value
+{
+ char *id;
+};
+
+DEFINE_HASHTABLE_INSERT(insert_some, struct key, struct value);
+DEFINE_HASHTABLE_SEARCH(search_some, struct key, struct value);
+DEFINE_HASHTABLE_REMOVE(remove_some, struct key, struct value);
+DEFINE_HASHTABLE_ITERATOR_SEARCH(search_itr_some, struct key);
+
+
+/*****************************************************************************/
+static unsigned int
+hashfromkey(void *ky)
+{
+ struct key *k = (struct key *)ky;
+ return (((k->one_ip << 17) | (k->one_ip >> 15)) ^ k->two_ip) +
+ (k->one_port * 17) + (k->two_port * 13 * 29);
+}
+
+static int
+equalkeys(void *k1, void *k2)
+{
+ return (0 == memcmp(k1,k2,sizeof(struct key)));
+}
+
+/*****************************************************************************/
+int
+main(int argc, char **argv)
+{
+ struct key *k, *kk;
+ struct value *v, *found;
+ struct hashtable *h;
+ struct hashtable_itr *itr;
+ int i;
+
+ h = create_hashtable(16, hashfromkey, equalkeys);
+ if (NULL == h) exit(-1); /*oom*/
+
+
+/*****************************************************************************/
+/* Insertion */
+ for (i = 0; i < ITEM_COUNT; i++)
+ {
+ k = (struct key *)malloc(sizeof(struct key));
+ if (NULL == k) {
+ printf("ran out of memory allocating a key\n");
+ return 1;
+ }
+ k->one_ip = 0xcfccee40 + i;
+ k->two_ip = 0xcf0cee67 - (5 * i);
+ k->one_port = 22 + (7 * i);
+ k->two_port = 5522 - (3 * i);
+
+ v = (struct value *)malloc(sizeof(struct value));
+ v->id = "a value";
+
+ if (!insert_some(h,k,v)) exit(-1); /*oom*/
+ }
+ printf("After insertion, hashtable contains %u items.\n",
+ hashtable_count(h));
+
+/*****************************************************************************/
+/* Hashtable search */
+ k = (struct key *)malloc(sizeof(struct key));
+ if (NULL == k) {
+ printf("ran out of memory allocating a key\n");
+ return 1;
+ }
+
+ for (i = 0; i < ITEM_COUNT; i++)
+ {
+ k->one_ip = 0xcfccee40 + i;
+ k->two_ip = 0xcf0cee67 - (5 * i);
+ k->one_port = 22 + (7 * i);
+ k->two_port = 5522 - (3 * i);
+
+ if (NULL == (found = search_some(h,k))) {
+ printf("BUG: key not found\n");
+ }
+ }
+
+/*****************************************************************************/
+/* Hashtable iteration */
+ /* Iterator constructor only returns a valid iterator if
+ * the hashtable is not empty */
+ itr = hashtable_iterator(h);
+ i = 0;
+ if (hashtable_count(h) > 0)
+ {
+ do {
+ kk = hashtable_iterator_key(itr);
+ v = hashtable_iterator_value(itr);
+ /* here (kk,v) are a valid (key, value) pair */
+ /* We could call 'hashtable_remove(h,kk)' - and this operation
+ * 'free's kk. However, the iterator is then broken.
+ * This is why hashtable_iterator_remove exists - see below.
+ */
+ i++;
+
+ } while (hashtable_iterator_advance(itr));
+ }
+ printf("Iterated through %u entries.\n", i);
+
+/*****************************************************************************/
+/* Hashtable iterator search */
+
+ /* Try the search some method */
+ for (i = 0; i < ITEM_COUNT; i++)
+ {
+ k->one_ip = 0xcfccee40 + i;
+ k->two_ip = 0xcf0cee67 - (5 * i);
+ k->one_port = 22 + (7 * i);
+ k->two_port = 5522 - (3 * i);
+
+ if (0 == search_itr_some(itr,h,k)) {
+ printf("BUG: key not found searching with iterator");
+ }
+ }
+
+/*****************************************************************************/
+/* Hashtable removal */
+
+ for (i = 0; i < ITEM_COUNT; i++)
+ {
+ k->one_ip = 0xcfccee40 + i;
+ k->two_ip = 0xcf0cee67 - (5 * i);
+ k->one_port = 22 + (7 * i);
+ k->two_port = 5522 - (3 * i);
+
+ if (NULL == (found = remove_some(h,k))) {
+ printf("BUG: key not found for removal\n");
+ }
+ }
+ printf("After removal, hashtable contains %u items.\n",
+ hashtable_count(h));
+
+/*****************************************************************************/
+/* Hashtable destroy and create */
+
+ hashtable_destroy(h, 1);
+ h = NULL;
+ free(k);
+
+ h = create_hashtable(160, hashfromkey, equalkeys);
+ if (NULL == h) {
+ printf("out of memory allocating second hashtable\n");
+ return 1;
+ }
+
+/*****************************************************************************/
+/* Hashtable insertion */
+
+ for (i = 0; i < ITEM_COUNT; i++)
+ {
+ k = (struct key *)malloc(sizeof(struct key));
+ k->one_ip = 0xcfccee40 + i;
+ k->two_ip = 0xcf0cee67 - (5 * i);
+ k->one_port = 22 + (7 * i);
+ k->two_port = 5522 - (3 * i);
+
+ v = (struct value *)malloc(sizeof(struct value));
+ v->id = "a value";
+
+ if (!insert_some(h,k,v))
+ {
+ printf("out of memory inserting into second hashtable\n");
+ return 1;
+ }
+ }
+ printf("After insertion, hashtable contains %u items.\n",
+ hashtable_count(h));
+
+/*****************************************************************************/
+/* Hashtable iterator search and iterator remove */
+
+ k = (struct key *)malloc(sizeof(struct key));
+ if (NULL == k) {
+ printf("ran out of memory allocating a key\n");
+ return 1;
+ }
+
+ for (i = ITEM_COUNT - 1; i >= 0; i = i - 7)
+ {
+ k->one_ip = 0xcfccee40 + i;
+ k->two_ip = 0xcf0cee67 - (5 * i);
+ k->one_port = 22 + (7 * i);
+ k->two_port = 5522 - (3 * i);
+
+ if (0 == search_itr_some(itr, h, k)) {
+ printf("BUG: key %u not found for search preremoval using iterator\n", i);
+ return 1;
+ }
+ if (0 == hashtable_iterator_remove(itr)) {
+ printf("BUG: key not found for removal using iterator\n");
+ return 1;
+ }
+ }
+ free(itr);
+
+/*****************************************************************************/
+/* Hashtable iterator remove and advance */
+
+ for (itr = hashtable_iterator(h);
+ hashtable_iterator_remove(itr) != 0; ) {
+ ;
+ }
+ free(itr);
+ printf("After removal, hashtable contains %u items.\n",
+ hashtable_count(h));
+
+/*****************************************************************************/
+/* Hashtable destroy */
+
+ hashtable_destroy(h, 1);
+ free(k);
+ return 0;
+}
+
+/*
+ * Copyright (c) 2002, 2004, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
diff --git a/runtime/hashtable_itr.c b/runtime/hashtable_itr.c
new file mode 100644
index 00000000..967287f1
--- /dev/null
+++ b/runtime/hashtable_itr.c
@@ -0,0 +1,190 @@
+/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+
+#include "hashtable.h"
+#include "hashtable_private.h"
+#include "hashtable_itr.h"
+#include <stdlib.h> /* defines NULL */
+
+/*****************************************************************************/
+/* hashtable_iterator - iterator constructor */
+
+struct hashtable_itr *
+hashtable_iterator(struct hashtable *h)
+{
+ unsigned int i, tablelength;
+ struct hashtable_itr *itr = (struct hashtable_itr *)
+ malloc(sizeof(struct hashtable_itr));
+ if (NULL == itr) return NULL;
+ itr->h = h;
+ itr->e = NULL;
+ itr->parent = NULL;
+ tablelength = h->tablelength;
+ itr->index = tablelength;
+ if (0 == h->entrycount) return itr;
+
+ for (i = 0; i < tablelength; i++)
+ {
+ if (NULL != h->table[i])
+ {
+ itr->e = h->table[i];
+ itr->index = i;
+ break;
+ }
+ }
+ return itr;
+}
+
+/*****************************************************************************/
+/* key - return the key of the (key,value) pair at the current position */
+/* value - return the value of the (key,value) pair at the current position */
+
+#if 0 /* these are now inline functions! */
+void *
+hashtable_iterator_key(struct hashtable_itr *i)
+{ return i->e->k; }
+
+void *
+hashtable_iterator_value(struct hashtable_itr *i)
+{ return i->e->v; }
+#endif
+
+/*****************************************************************************/
+/* advance - advance the iterator to the next element
+ * returns zero if advanced to end of table */
+
+int
+hashtable_iterator_advance(struct hashtable_itr *itr)
+{
+ unsigned int j,tablelength;
+ struct entry **table;
+ struct entry *next;
+ if (NULL == itr->e) return 0; /* stupidity check */
+
+ next = itr->e->next;
+ if (NULL != next)
+ {
+ itr->parent = itr->e;
+ itr->e = next;
+ return -1;
+ }
+ tablelength = itr->h->tablelength;
+ itr->parent = NULL;
+ if (tablelength <= (j = ++(itr->index)))
+ {
+ itr->e = NULL;
+ return 0;
+ }
+ table = itr->h->table;
+ while (NULL == (next = table[j]))
+ {
+ if (++j >= tablelength)
+ {
+ itr->index = tablelength;
+ itr->e = NULL;
+ return 0;
+ }
+ }
+ itr->index = j;
+ itr->e = next;
+ return -1;
+}
+
+/*****************************************************************************/
+/* remove - remove the entry at the current iterator position
+ * and advance the iterator, if there is a successive
+ * element.
+ * If you want the value, read it before you remove:
+ * beware memory leaks if you don't.
+ * Returns zero if end of iteration. */
+
+int
+hashtable_iterator_remove(struct hashtable_itr *itr)
+{
+ struct entry *remember_e, *remember_parent;
+ int ret;
+
+ /* Do the removal */
+ if (NULL == (itr->parent))
+ {
+ /* element is head of a chain */
+ itr->h->table[itr->index] = itr->e->next;
+ } else {
+ /* element is mid-chain */
+ itr->parent->next = itr->e->next;
+ }
+ /* itr->e is now outside the hashtable */
+ remember_e = itr->e;
+ itr->h->entrycount--;
+ freekey(remember_e->k);
+
+ /* Advance the iterator, correcting the parent */
+ remember_parent = itr->parent;
+ ret = hashtable_iterator_advance(itr);
+ if (itr->parent == remember_e) { itr->parent = remember_parent; }
+ free(remember_e);
+ return ret;
+}
+
+/*****************************************************************************/
+int /* returns zero if not found */
+hashtable_iterator_search(struct hashtable_itr *itr,
+ struct hashtable *h, void *k)
+{
+ struct entry *e, *parent;
+ unsigned int hashvalue, index;
+
+ hashvalue = hash(h,k);
+ index = indexFor(h->tablelength,hashvalue);
+
+ e = h->table[index];
+ parent = NULL;
+ while (NULL != e)
+ {
+ /* Check hash value to short circuit heavier comparison */
+ if ((hashvalue == e->h) && (h->eqfn(k, e->k)))
+ {
+ itr->index = index;
+ itr->e = e;
+ itr->parent = parent;
+ itr->h = h;
+ return -1;
+ }
+ parent = e;
+ e = e->next;
+ }
+ return 0;
+}
+
+
+/*
+ * Copyright (c) 2002, 2004, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
diff --git a/runtime/hashtable_itr.h b/runtime/hashtable_itr.h
new file mode 100644
index 00000000..1c206b6e
--- /dev/null
+++ b/runtime/hashtable_itr.h
@@ -0,0 +1,112 @@
+/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+
+#ifndef __HASHTABLE_ITR_CWC22__
+#define __HASHTABLE_ITR_CWC22__
+#include "hashtable.h"
+#include "hashtable_private.h" /* needed to enable inlining */
+
+/*****************************************************************************/
+/* This struct is only concrete here to allow the inlining of two of the
+ * accessor functions. */
+struct hashtable_itr
+{
+ struct hashtable *h;
+ struct entry *e;
+ struct entry *parent;
+ unsigned int index;
+};
+
+
+/*****************************************************************************/
+/* hashtable_iterator
+ */
+
+struct hashtable_itr *
+hashtable_iterator(struct hashtable *h);
+
+/*****************************************************************************/
+/* hashtable_iterator_key
+ * - return the value of the (key,value) pair at the current position */
+
+static inline void *
+hashtable_iterator_key(struct hashtable_itr *i)
+{
+ return i->e->k;
+}
+
+/*****************************************************************************/
+/* value - return the value of the (key,value) pair at the current position */
+
+static inline void *
+hashtable_iterator_value(struct hashtable_itr *i)
+{
+ return i->e->v;
+}
+
+/*****************************************************************************/
+/* advance - advance the iterator to the next element
+ * returns zero if advanced to end of table */
+
+int
+hashtable_iterator_advance(struct hashtable_itr *itr);
+
+/*****************************************************************************/
+/* remove - remove current element and advance the iterator to the next element
+ * NB: if you need the value to free it, read it before
+ * removing. ie: beware memory leaks!
+ * returns zero if advanced to end of table */
+
+int
+hashtable_iterator_remove(struct hashtable_itr *itr);
+
+/*****************************************************************************/
+/* search - overwrite the supplied iterator, to point to the entry
+ * matching the supplied key.
+ h points to the hashtable to be searched.
+ * returns zero if not found. */
+int
+hashtable_iterator_search(struct hashtable_itr *itr,
+ struct hashtable *h, void *k);
+
+#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \
+int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \
+{ \
+ return (hashtable_iterator_search(i,h,k)); \
+}
+
+
+
+#endif /* __HASHTABLE_ITR_CWC22__*/
+
+/*
+ * Copyright (c) 2002, 2004, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
diff --git a/runtime/hashtable_private.h b/runtime/hashtable_private.h
new file mode 100644
index 00000000..10b82da4
--- /dev/null
+++ b/runtime/hashtable_private.h
@@ -0,0 +1,86 @@
+/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
+
+#ifndef __HASHTABLE_PRIVATE_CWC22_H__
+#define __HASHTABLE_PRIVATE_CWC22_H__
+
+#include "hashtable.h"
+
+/*****************************************************************************/
+struct entry
+{
+ void *k, *v;
+ unsigned int h;
+ struct entry *next;
+};
+
+struct hashtable {
+ unsigned int tablelength;
+ struct entry **table;
+ unsigned int entrycount;
+ unsigned int loadlimit;
+ unsigned int primeindex;
+ unsigned int (*hashfn) (void *k);
+ int (*eqfn) (void *k1, void *k2);
+ void (*dest) (void *v); /* destructor for values, if NULL use free() */
+};
+
+/*****************************************************************************/
+unsigned int
+hash(struct hashtable *h, void *k);
+
+/*****************************************************************************/
+/* indexFor */
+static inline unsigned int
+indexFor(unsigned int tablelength, unsigned int hashvalue) {
+ return (hashvalue % tablelength);
+};
+
+/* Only works if tablelength == 2^N */
+/*static inline unsigned int
+indexFor(unsigned int tablelength, unsigned int hashvalue)
+{
+ return (hashvalue & (tablelength - 1u));
+}
+*/
+
+/*****************************************************************************/
+#define freekey(X) free(X)
+/*define freekey(X) ; */
+
+
+/*****************************************************************************/
+
+#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/
+
+/*
+ * Copyright (c) 2002, Christopher Clark
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
diff --git a/runtime/module-template.h b/runtime/module-template.h
index 3e963199..c2585e67 100644
--- a/runtime/module-template.h
+++ b/runtime/module-template.h
@@ -46,6 +46,12 @@
DEFobjCurrIf(obj)
#define DEF_LMOD_STATIC_DATA \
DEF_MOD_STATIC_DATA
+#define DEF_PMOD_STATIC_DATA \
+ DEFobjCurrIf(obj) \
+ DEF_MOD_STATIC_DATA
+#define DEF_SMOD_STATIC_DATA \
+ DEFobjCurrIf(obj) \
+ DEF_MOD_STATIC_DATA
/* Macro to define the module type. Each module can only have a single type. If
@@ -65,10 +71,22 @@ static rsRetVal modGetType(eModType_t *modType) \
#define MODULE_TYPE_INPUT MODULE_TYPE(eMOD_IN)
#define MODULE_TYPE_OUTPUT MODULE_TYPE(eMOD_OUT)
+#define MODULE_TYPE_PARSER MODULE_TYPE(eMOD_PARSER)
+#define MODULE_TYPE_STRGEN MODULE_TYPE(eMOD_STRGEN)
#define MODULE_TYPE_LIB \
DEF_LMOD_STATIC_DATA \
MODULE_TYPE(eMOD_LIB)
+/* Macro to define whether the module should be kept dynamically linked.
+ */
+#define MODULE_KEEP_TYPE(x)\
+static rsRetVal modGetKeepType(eModKeepType_t *modKeepType) \
+ { \
+ *modKeepType = x; \
+ return RS_RET_OK;\
+ }
+#define MODULE_TYPE_NOKEEP MODULE_KEEP_TYPE(eMOD_NOKEEP)
+#define MODULE_TYPE_KEEP MODULE_KEEP_TYPE(eMOD_KEEP)
/* macro to define a unique module id. This must be able to fit in a void*. The
* module id must be unique inside a running rsyslogd application. It is used to
@@ -334,6 +352,8 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\
*pEtryPoint = modGetID;\
} else if(!strcmp((char*) name, "getType")) {\
*pEtryPoint = modGetType;\
+ } else if(!strcmp((char*) name, "getKeepType")) {\
+ *pEtryPoint = modGetKeepType;\
}
/* the following definition is the standard block for queryEtryPt for output
@@ -368,6 +388,17 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\
*pEtryPoint = endTransaction;\
}
+
+/* the following definition is a queryEtryPt block that must be added
+ * if a non-output module supports "isCompatibleWithFeature".
+ * rgerhards, 2009-07-20
+ */
+#define CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES \
+ else if(!strcmp((char*) name, "isCompatibleWithFeature")) {\
+ *pEtryPoint = isCompatibleWithFeature;\
+ }
+
+
/* the following definition is the standard block for queryEtryPt for INPUT
* modules. This can be used if no specific handling (e.g. to cover version
* differences) is needed.
@@ -389,6 +420,30 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\
#define CODEqueryEtryPt_STD_LIB_QUERIES \
CODEqueryEtryPt_STD_MOD_QUERIES
+/* the following definition is the standard block for queryEtryPt for PARSER
+ * modules. This can be used if no specific handling (e.g. to cover version
+ * differences) is needed.
+ */
+#define CODEqueryEtryPt_STD_PMOD_QUERIES \
+ CODEqueryEtryPt_STD_MOD_QUERIES \
+ else if(!strcmp((char*) name, "parse")) {\
+ *pEtryPoint = parse;\
+ } else if(!strcmp((char*) name, "GetParserName")) {\
+ *pEtryPoint = GetParserName;\
+ }
+
+/* the following definition is the standard block for queryEtryPt for Strgen
+ * modules. This can be used if no specific handling (e.g. to cover version
+ * differences) is needed.
+ */
+#define CODEqueryEtryPt_STD_SMOD_QUERIES \
+ CODEqueryEtryPt_STD_MOD_QUERIES \
+ else if(!strcmp((char*) name, "strgen")) {\
+ *pEtryPoint = strgen;\
+ } else if(!strcmp((char*) name, "GetName")) {\
+ *pEtryPoint = GetStrgenName;\
+ }
+
/* modInit()
* This has an extra parameter, which is the specific name of the modInit
* function. That is needed for built-in modules, which must have unique
@@ -579,5 +634,58 @@ static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\
}
+/* parse() - main entry point of parser modules
+ */
+#define BEGINparse \
+static rsRetVal parse(msg_t *pMsg)\
+{\
+ DEFiRet;
+
+#define CODESTARTparse \
+ assert(pMsg != NULL);
+
+#define ENDparse \
+ RETiRet;\
+}
+
+
+/* strgen() - main entry point of parser modules
+ */
+#define BEGINstrgen \
+static rsRetVal strgen(msg_t *pMsg, uchar **ppBuf, size_t *pLenBuf) \
+{\
+ DEFiRet;
+
+#define CODESTARTstrgen \
+ assert(pMsg != NULL);
+
+#define ENDstrgen \
+ RETiRet;\
+}
+
+
+/* function to specify the parser name. This is done via a single command which
+ * receives a ANSI string as parameter.
+ */
+#define PARSER_NAME(x) \
+static rsRetVal GetParserName(uchar **ppSz)\
+{\
+ *ppSz = UCHAR_CONSTANT(x);\
+ return RS_RET_OK;\
+}
+
+
+
+/* function to specify the strgen name. This is done via a single command which
+ * receives a ANSI string as parameter.
+ */
+#define STRGEN_NAME(x) \
+static rsRetVal GetStrgenName(uchar **ppSz)\
+{\
+ *ppSz = UCHAR_CONSTANT(x);\
+ return RS_RET_OK;\
+}
+
+
/* vim:set ai:
*/
diff --git a/runtime/modules.c b/runtime/modules.c
index 871f356a..4541bddf 100644
--- a/runtime/modules.c
+++ b/runtime/modules.c
@@ -11,7 +11,7 @@
*
* File begun on 2007-07-22 by RGerhards
*
- * Copyright 2007 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -57,10 +57,14 @@
#include "cfsysline.h"
#include "modules.h"
#include "errmsg.h"
+#include "parser.h"
+#include "strgen.h"
/* static data */
DEFobjStaticHelpers
DEFobjCurrIf(errmsg)
+DEFobjCurrIf(parser)
+DEFobjCurrIf(strgen)
/* we must ensure that only one thread at one time tries to load or unload
* modules, otherwise we may see race conditions. This first came up with
@@ -73,10 +77,33 @@ static pthread_mutex_t mutLoadUnload;
static modInfo_t *pLoadedModules = NULL; /* list of currently-loaded modules */
static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */
+/* already dlopen()-ed libs */
+static struct dlhandle_s *pHandles = NULL;
+
/* config settings */
uchar *pModDir = NULL; /* read-only after startup */
+/* we provide a set of dummy functions for modules that do not support the
+ * some interfaces.
+ * On the commit feature: As the modules do not support it, they commit each message they
+ * receive, and as such the dummies can always return RS_RET_OK without causing
+ * harm. This simplifies things as in action processing we do not need to check
+ * if the transactional entry points exist.
+ */
+static rsRetVal dummyBeginTransaction()
+{
+ return RS_RET_OK;
+}
+static rsRetVal dummyEndTransaction()
+{
+ return RS_RET_OK;
+}
+static rsRetVal dummyIsCompatibleWithFeature()
+{
+ return RS_RET_INCOMPATIBLE;
+}
+
#ifdef DEBUG
/* we add some home-grown support to track our users (and detect who does not free us). In
* the long term, this should probably be migrated into debug.c (TODO). -- rgerhards, 2008-03-11
@@ -208,7 +235,9 @@ static void moduleDestruct(modInfo_t *pThis)
# ifdef VALGRIND
# warning "dlclose disabled for valgrind"
# else
- dlclose(pThis->pModHdlr);
+ if (pThis->eKeepType == eMOD_NOKEEP) {
+ dlclose(pThis->pModHdlr);
+ }
# endif
}
@@ -216,19 +245,38 @@ static void moduleDestruct(modInfo_t *pThis)
}
+/* This enables a module to query the core for specific features.
+ * rgerhards, 2009-04-22
+ */
+static rsRetVal queryCoreFeatureSupport(int *pBool, unsigned uFeat)
+{
+ DEFiRet;
+
+ if((pBool == NULL))
+ ABORT_FINALIZE(RS_RET_PARAM_ERROR);
+
+ *pBool = (uFeat & CORE_FEATURE_BATCHING) ? 1 : 0;
+
+finalize_it:
+ RETiRet;
+}
+
+
/* The following function is the queryEntryPoint for host-based entry points.
* Modules may call it to get access to core interface functions. Please note
* that utility functions can be accessed via shared libraries - at least this
* is my current shool of thinking.
* Please note that the implementation as a query interface allows to take
* care of plug-in interface version differences. -- rgerhards, 2007-07-31
+ * ... but often it better not to use a new interface. So we now add core
+ * functions here that a plugin may request. -- rgerhards, 2009-04-22
*/
static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)())
{
DEFiRet;
if((name == NULL) || (pEtryPoint == NULL))
- return RS_RET_PARAM_ERROR;
+ ABORT_FINALIZE(RS_RET_PARAM_ERROR);
if(!strcmp((char*) name, "regCfSysLineHdlr")) {
*pEtryPoint = regCfSysLineHdlr;
@@ -236,6 +284,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)())
*pEtryPoint = objGetObjInterface;
} else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) {
*pEtryPoint = OMSRgetSupportedTplOpts;
+ } else if(!strcmp((char*) name, "queryCoreFeatureSupport")) {
+ *pEtryPoint = queryCoreFeatureSupport;
} else {
*pEtryPoint = NULL; /* to be on the safe side */
ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND);
@@ -361,10 +411,16 @@ finalize_it:
static rsRetVal
doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr)
{
- DEFiRet;
rsRetVal localRet;
modInfo_t *pNew = NULL;
+ uchar *pName;
+ parser_t *pParser; /* used for parser modules */
+ strgen_t *pStrgen; /* used for strgen modules */
+ rsRetVal (*GetName)(uchar**);
rsRetVal (*modGetType)(eModType_t *pType);
+ rsRetVal (*modGetKeepType)(eModKeepType_t *pKeepType);
+ struct dlhandle_s *pHandle = NULL;
+ DEFiRet;
assert(modInit != NULL);
@@ -384,6 +440,8 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_
*/
CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getType", &modGetType));
CHKiRet((*modGetType)(&pNew->eType));
+ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getKeepType", &modGetKeepType));
+ CHKiRet((*modGetKeepType)(&pNew->eKeepType));
dbgprintf("module of type %d being loaded.\n", pNew->eType);
/* OK, we know we can successfully work with the module. So we now fill the
@@ -392,6 +450,11 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_
*/
CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modGetID", &pNew->modGetID));
CHKiRet((*pNew->modQueryEtryPt)((uchar*)"modExit", &pNew->modExit));
+ localRet = (*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature);
+ if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND)
+ pNew->isCompatibleWithFeature = dummyIsCompatibleWithFeature;
+ else if(localRet != RS_RET_OK)
+ ABORT_FINALIZE(localRet);
/* ... and now the module-specific interfaces */
switch(pNew->eType) {
@@ -406,25 +469,105 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_
CHKiRet((*pNew->modQueryEtryPt)((uchar*)"dbgPrintInstInfo", &pNew->dbgPrintInstInfo));
CHKiRet((*pNew->modQueryEtryPt)((uchar*)"doAction", &pNew->mod.om.doAction));
CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct));
- CHKiRet((*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature));
CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume));
/* try load optional interfaces */
localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP);
if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND)
ABORT_FINALIZE(localRet);
+
+ localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction);
+ if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND)
+ pNew->mod.om.beginTransaction = dummyBeginTransaction;
+ else if(localRet != RS_RET_OK)
+ ABORT_FINALIZE(localRet);
+
+ localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction);
+ if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) {
+ pNew->mod.om.endTransaction = dummyEndTransaction;
+ } else if(localRet != RS_RET_OK) {
+ ABORT_FINALIZE(localRet);
+ }
break;
case eMOD_LIB:
break;
+ case eMOD_PARSER:
+ /* first, we need to obtain the parser object. We could not do that during
+ * init as that would have caused class bootstrap issues which are not
+ * absolutely necessary. Note that we can call objUse() multiple times, it
+ * handles that.
+ */
+ CHKiRet(objUse(parser, CORE_COMPONENT));
+ /* here, we create a new parser object */
+ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parse", &pNew->mod.pm.parse));
+ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"GetParserName", &GetName));
+ CHKiRet(GetName(&pName));
+ CHKiRet(parser.Construct(&pParser));
+
+ /* check some features */
+ localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticSanitazion);
+ if(localRet == RS_RET_OK){
+ CHKiRet(parser.SetDoSanitazion(pParser, TRUE));
+ }
+ localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticPRIParsing);
+ if(localRet == RS_RET_OK){
+ CHKiRet(parser.SetDoPRIParsing(pParser, TRUE));
+ }
+
+ CHKiRet(parser.SetName(pParser, pName));
+ CHKiRet(parser.SetModPtr(pParser, pNew));
+ CHKiRet(parser.ConstructFinalize(pParser));
+ break;
+ case eMOD_STRGEN:
+ /* first, we need to obtain the strgen object. We could not do that during
+ * init as that would have caused class bootstrap issues which are not
+ * absolutely necessary. Note that we can call objUse() multiple times, it
+ * handles that.
+ */
+ CHKiRet(objUse(strgen, CORE_COMPONENT));
+ /* here, we create a new parser object */
+ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"strgen", &pNew->mod.sm.strgen));
+ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"GetName", &GetName));
+ CHKiRet(GetName(&pName));
+ CHKiRet(strgen.Construct(&pStrgen));
+ CHKiRet(strgen.SetName(pStrgen, pName));
+ CHKiRet(strgen.SetModPtr(pStrgen, pNew));
+ CHKiRet(strgen.ConstructFinalize(pStrgen));
+ break;
}
pNew->pszName = (uchar*) strdup((char*)name); /* we do not care if strdup() fails, we can accept that */
pNew->pModHdlr = pModHdlr;
/* TODO: take this from module */
- if(pModHdlr == NULL)
+ if(pModHdlr == NULL) {
pNew->eLinkType = eMOD_LINK_STATIC;
- else
+ } else {
pNew->eLinkType = eMOD_LINK_DYNAMIC_LOADED;
+ /* if we need to keep the linked module, save it */
+ if (pNew->eKeepType == eMOD_KEEP) {
+ /* see if we have this one already */
+ for (pHandle = pHandles; pHandle; pHandle = pHandle->next) {
+ if (!strcmp((char *)name, (char *)pHandle->pszName))
+ break;
+ }
+
+ /* not found, create it */
+ if (!pHandle) {
+ if((pHandle = malloc(sizeof (*pHandle))) == NULL) {
+ ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
+ }
+ if((pHandle->pszName = (uchar*) strdup((char*)name)) == NULL) {
+ free(pHandle);
+ ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
+ }
+ pHandle->pModHdlr = pModHdlr;
+ pHandle->next = pHandles;
+
+ pHandles = pHandle;
+ }
+ }
+ }
+
/* we initialized the structure, now let's add it to the linked list of modules */
addModToList(pNew);
@@ -461,14 +604,49 @@ static void modPrintList(void)
case eMOD_LIB:
dbgprintf("library");
break;
+ case eMOD_PARSER:
+ dbgprintf("parser");
+ break;
+ case eMOD_STRGEN:
+ dbgprintf("strgen");
+ break;
}
dbgprintf(" module.\n");
dbgprintf("Entry points:\n");
dbgprintf("\tqueryEtryPt: 0x%lx\n", (unsigned long) pMod->modQueryEtryPt);
- dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction);
- dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct);
dbgprintf("\tdbgPrintInstInfo: 0x%lx\n", (unsigned long) pMod->dbgPrintInstInfo);
dbgprintf("\tfreeInstance: 0x%lx\n", (unsigned long) pMod->freeInstance);
+ switch(pMod->eType) {
+ case eMOD_OUT:
+ dbgprintf("Output Module Entry Points:\n");
+ dbgprintf("\tdoAction: 0x%lx\n", (unsigned long) pMod->mod.om.doAction);
+ dbgprintf("\tparseSelectorAct: 0x%lx\n", (unsigned long) pMod->mod.om.parseSelectorAct);
+ dbgprintf("\ttryResume: 0x%lx\n", (unsigned long) pMod->tryResume);
+ dbgprintf("\tdoHUP: 0x%lx\n", (unsigned long) pMod->doHUP);
+ dbgprintf("\tBeginTransaction: 0x%lx\n", (unsigned long)
+ ((pMod->mod.om.beginTransaction == dummyBeginTransaction) ?
+ 0 : pMod->mod.om.beginTransaction));
+ dbgprintf("\tEndTransaction: 0x%lx\n", (unsigned long)
+ ((pMod->mod.om.endTransaction == dummyEndTransaction) ?
+ 0 : pMod->mod.om.endTransaction));
+ break;
+ case eMOD_IN:
+ dbgprintf("Input Module Entry Points\n");
+ dbgprintf("\trunInput: 0x%lx\n", (unsigned long) pMod->mod.im.runInput);
+ dbgprintf("\twillRun: 0x%lx\n", (unsigned long) pMod->mod.im.willRun);
+ dbgprintf("\tafterRun: 0x%lx\n", (unsigned long) pMod->mod.im.afterRun);
+ break;
+ case eMOD_LIB:
+ break;
+ case eMOD_PARSER:
+ dbgprintf("Parser Module Entry Points\n");
+ dbgprintf("\tparse: 0x%lx\n", (unsigned long) pMod->mod.pm.parse);
+ break;
+ case eMOD_STRGEN:
+ dbgprintf("Strgen Module Entry Points\n");
+ dbgprintf("\tstrgen: 0x%lx\n", (unsigned long) pMod->mod.sm.strgen);
+ break;
+ }
dbgprintf("\n");
pMod = GetNxt(pMod); /* done, go next */
}
@@ -596,6 +774,7 @@ Load(uchar *pModName)
modInfo_t *pModInfo;
uchar *pModDirCurr, *pModDirNext;
int iLoadCnt;
+ struct dlhandle_s *pHandle = NULL;
assert(pModName != NULL);
dbgprintf("Requested to load module '%s'\n", pModName);
@@ -685,7 +864,20 @@ Load(uchar *pModName)
/* complete load path constructed, so ... GO! */
dbgprintf("loading module '%s'\n", szPath);
- pModHdlr = dlopen((char *) szPath, RTLD_NOW);
+
+ /* see if we have this one already */
+ for (pHandle = pHandles; pHandle; pHandle = pHandle->next) {
+ if (!strcmp((char *)pModName, (char *)pHandle->pszName)) {
+ pModHdlr = pHandle->pModHdlr;
+ break;
+ }
+ }
+
+ /* not found, try to dynamically link it */
+ if (!pModHdlr) {
+ pModHdlr = dlopen((char *) szPath, RTLD_NOW);
+ }
+
iLoadCnt++;
} while(pModHdlr == NULL && *pModName != '/' && pModDirNext);
@@ -807,6 +999,7 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA
CODESTARTObjClassExit(module)
/* release objects we no longer need */
objRelease(errmsg, CORE_COMPONENT);
+ objRelease(parser, CORE_COMPONENT);
/* We have a problem in our reference counting, which leads to this function
* being called too early. This usually is no problem, but if we destroy
* the mutex object, we get into trouble. So rather than finding the root cause,
diff --git a/runtime/modules.h b/runtime/modules.h
index 4d874019..4daaf1f9 100644
--- a/runtime/modules.h
+++ b/runtime/modules.h
@@ -12,7 +12,7 @@
*
* File begun on 2007-07-22 by RGerhards
*
- * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -51,9 +51,11 @@
#define CURR_MOD_IF_VERSION 5
typedef enum eModType_ {
- eMOD_IN, /* input module */
- eMOD_OUT, /* output module */
- eMOD_LIB /* library module - this module provides one or many interfaces */
+ eMOD_IN = 0, /* input module */
+ eMOD_OUT = 1, /* output module */
+ eMOD_LIB = 2, /* library module */
+ eMOD_PARSER = 3,/* parser module */
+ eMOD_STRGEN = 4 /* strgen module */
} eModType_t;
@@ -73,12 +75,26 @@ typedef enum eModLinkType_ {
eMOD_LINK_ALL /* special: all linkage types, e.g. for unload */
} eModLinkType_t;
-typedef struct modInfo_s {
+/* remember which shared libs we dlopen()-ed */
+struct dlhandle_s {
+ uchar *pszName;
+ void *pModHdlr;
+ struct dlhandle_s *next;
+};
+
+/* should this module be kept linked? */
+typedef enum eModKeepType_ {
+ eMOD_NOKEEP,
+ eMOD_KEEP
+} eModKeepType_t;
+
+struct modInfo_s {
struct modInfo_s *pPrev; /* support for creating a double linked module list */
struct modInfo_s *pNext; /* support for creating a linked module list */
int iIFVers; /* Interface version of module */
eModType_t eType; /* type of this module */
eModLinkType_t eLinkType;
+ eModKeepType_t eKeepType; /* keep the module dynamically linked on unload */
uchar* pszName; /* printable module name, e.g. for dbgprintf */
unsigned uRefCnt; /* reference count for this module; 0 -> may be unloaded */
/* functions supported by all types of modules */
@@ -111,11 +127,20 @@ typedef struct modInfo_s {
struct {/* data for output modules */
/* below: perform the configured action
*/
+ rsRetVal (*beginTransaction)(void*);
rsRetVal (*doAction)(uchar**, unsigned, void*);
+ rsRetVal (*endTransaction)(void*);
rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**);
} om;
struct { /* data for library modules */
- } fm;
+ char dummy;
+ } lm;
+ struct { /* data for parser modules */
+ rsRetVal (*parse)(msg_t*);
+ } pm;
+ struct { /* data for strgen modules */
+ rsRetVal (*strgen)(msg_t*, uchar**, size_t *);
+ } sm;
} mod;
void *pModHdlr; /* handler to the dynamic library holding the module */
# ifdef DEBUG
@@ -124,7 +149,8 @@ typedef struct modInfo_s {
*/
modUsr_t *pModUsrRoot;
# endif
-} modInfo_t;
+};
+
/* interfaces */
BEGINinterface(module) /* name must also be changed in ENDinterface macro! */
@@ -148,8 +174,4 @@ PROTOTYPEObj(module);
/* TODO: remove them below (means move the config init code) -- rgerhards, 2008-02-19 */
extern uchar *pModDir; /* read-only after startup */
-
#endif /* #ifndef MODULES_H_INCLUDED */
-/*
- * vi:set ai:
- */
diff --git a/runtime/msg.c b/runtime/msg.c
index 6335f462..0744edd5 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -35,6 +35,8 @@
#include <string.h>
#include <assert.h>
#include <ctype.h>
+#include <sys/socket.h>
+#include <netdb.h>
#if HAVE_MALLOC_H
# include <malloc.h>
#endif
@@ -51,6 +53,7 @@
#include "unicode-helper.h"
#include "ruleset.h"
#include "prop.h"
+#include "net.h"
/* static data */
DEFobjStaticHelpers
@@ -59,6 +62,7 @@ DEFobjCurrIf(datetime)
DEFobjCurrIf(glbl)
DEFobjCurrIf(regexp)
DEFobjCurrIf(prop)
+DEFobjCurrIf(net)
static struct {
uchar *pszName;
@@ -274,8 +278,128 @@ static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warn
static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14",
"15", "16", "17", "18", "19", "20", "21", "22", "23" };
+/* global variables */
+#if defined(HAVE_MALLOC_TRIM) && !defined(HAVE_ATOMIC_BUILTINS)
+static pthread_mutex_t mutTrimCtr; /* mutex to handle malloc trim */
+#endif
+
/* some forward declarations */
-static int getAPPNAMELen(msg_t *pM, bool bLockMutex);
+static int getAPPNAMELen(msg_t *pM, sbool bLockMutex);
+
+
+/* The following functions will support advanced output module
+ * multithreading, once this is implemented. Currently, we
+ * include them as hooks only. The idea is that we need to guard
+ * some msg objects data fields against concurrent access if
+ * we run on multiple threads. Please note that in any case this
+ * is not necessary for calls from INPUT modules, because they
+ * construct the message object and do this serially. Only when
+ * the message is in the processing queue, multiple threads may
+ * access a single object. Consequently, there are no guard functions
+ * for "set" methods, as these are called during input. Only "get"
+ * functions that modify important structures have them.
+ * rgerhards, 2007-07-20
+ * We now support locked and non-locked operations, depending on
+ * the configuration of rsyslog. To support this, we use function
+ * pointers. Initially, we start in non-locked mode. There, all
+ * locking operations call into dummy functions. When locking is
+ * enabled, the function pointers are changed to functions doing
+ * actual work. We also introduced another MsgPrepareEnqueue() function
+ * which initializes the locking structures, if needed. This is
+ * necessary because internal messages during config file startup
+ * processing are always created in non-locking mode. So we can
+ * not initialize locking structures during constructions. We now
+ * postpone this until when the message is fully constructed and
+ * enqueued. Then we know the status of locking. This has a nice
+ * side effect, and that is that during the initial creation of
+ * the Msg object no locking needs to be done, which results in better
+ * performance. -- rgerhards, 2008-01-05
+ */
+static void (*funcLock)(msg_t *pMsg);
+static void (*funcUnlock)(msg_t *pMsg);
+static void (*funcDeleteMutex)(msg_t *pMsg);
+void (*funcMsgPrepareEnqueue)(msg_t *pMsg);
+#if 1 /* This is a debug aid */
+#define MsgLock(pMsg) funcLock(pMsg)
+#define MsgUnlock(pMsg) funcUnlock(pMsg)
+#else
+#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; }
+#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); }
+#endif
+
+/* the next function is a dummy to be used by the looking functions
+ * when the class is not yet running in an environment where locking
+ * is necessary. Please note that the need to lock can (and will) change
+ * during a single run. Typically, this is depending on the operation mode
+ * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05
+ */
+static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg)
+{
+ /* empty be design */
+}
+
+
+/* The following function prepares a message for enqueue into the queue. This is
+ * where a message may be accessed by multiple threads. This implementation here
+ * is the version for multiple concurrent acces. It initializes the locking
+ * structures.
+ * TODO: change to an iRet interface! -- rgerhards, 2008-07-14
+ */
+static void MsgPrepareEnqueueLockingCase(msg_t *pThis)
+{
+ BEGINfunc
+ assert(pThis != NULL);
+ pthread_mutex_init(&pThis->mut, NULL);
+ pThis->bDoLock = 1;
+ ENDfunc
+}
+
+
+/* ... and now the locking and unlocking implementations: */
+static void MsgLockLockingCase(msg_t *pThis)
+{
+ /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */
+ assert(pThis != NULL);
+ if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */
+ pthread_mutex_lock(&pThis->mut);
+}
+
+static void MsgUnlockLockingCase(msg_t *pThis)
+{
+ /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */
+ assert(pThis != NULL);
+ if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */
+ pthread_mutex_unlock(&pThis->mut);
+}
+
+/* delete the mutex object on message destruction (locking case)
+ */
+static void MsgDeleteMutexLockingCase(msg_t *pThis)
+{
+ assert(pThis != NULL);
+ pthread_mutex_destroy(&pThis->mut);
+}
+
+/* enable multiple concurrent access on the message object
+ * This works on a class-wide basis and can bot be undone.
+ * That is, if it is once enabled, it can not be disabled during
+ * the same run. When this function is called, no other thread
+ * must manipulate message objects. Then we would have race conditions,
+ * but guarding against this is counter-productive because it
+ * would cost additional time. Plus, it would be a programming error.
+ * rgerhards, 2008-01-05
+ */
+rsRetVal MsgEnableThreadSafety(void)
+{
+ DEFiRet;
+ funcLock = MsgLockLockingCase;
+ funcUnlock = MsgUnlockLockingCase;
+ funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase;
+ funcDeleteMutex = MsgDeleteMutexLockingCase;
+ RETiRet;
+}
+
+/* end locking functions */
static inline int getProtocolVersion(msg_t *pM)
@@ -284,11 +408,48 @@ static inline int getProtocolVersion(msg_t *pM)
}
+/* do a DNS reverse resolution, if not already done, reflect status
+ * rgerhards, 2009-11-16
+ */
+static inline rsRetVal
+resolveDNS(msg_t *pMsg) {
+ rsRetVal localRet;
+ prop_t *propFromHost = NULL;
+ prop_t *propFromHostIP = NULL;
+ uchar fromHost[NI_MAXHOST];
+ uchar fromHostIP[NI_MAXHOST];
+ uchar fromHostFQDN[NI_MAXHOST];
+ DEFiRet;
+
+ MsgLock(pMsg);
+ CHKiRet(objUse(net, CORE_COMPONENT));
+ if(pMsg->msgFlags & NEEDS_DNSRESOL) {
+ localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP);
+ if(localRet == RS_RET_OK) {
+ MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost);
+ CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP));
+ }
+ }
+finalize_it:
+ MsgUnlock(pMsg);
+ if(iRet != RS_RET_OK) {
+ /* best we can do: remove property */
+ MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost);
+ prop.Destruct(&propFromHost);
+ }
+ if(propFromHost != NULL)
+ prop.Destruct(&propFromHost);
+ if(propFromHostIP != NULL)
+ prop.Destruct(&propFromHostIP);
+ RETiRet;
+}
+
+
static inline void
getInputName(msg_t *pM, uchar **ppsz, int *plen)
{
BEGINfunc
- if(pM == NULL) {
+ if(pM == NULL || pM->pInputName == NULL) {
*ppsz = UCHAR_CONSTANT("");
*plen = 0;
} else {
@@ -307,6 +468,7 @@ getRcvFromIP(msg_t *pM)
if(pM == NULL) {
psz = UCHAR_CONSTANT("");
} else {
+ resolveDNS(pM); /* make sure we have a resolved entry */
if(pM->pRcvFromIP == NULL)
psz = UCHAR_CONSTANT("");
else
@@ -399,6 +561,8 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID)
*pPropID = PROP_SYS_MINUTE;
} else if(!strcmp((char*) pName, "$myhostname")) {
*pPropID = PROP_SYS_MYHOSTNAME;
+ } else if(!strcmp((char*) pName, "$bom")) {
+ *pPropID = PROP_SYS_BOM;
} else {
*pPropID = PROP_INVALID;
iRet = RS_RET_VAR_NOT_FOUND;
@@ -480,127 +644,14 @@ uchar *propIDToName(propid_t propID)
return UCHAR_CONSTANT("$MINUTE");
case PROP_SYS_MYHOSTNAME:
return UCHAR_CONSTANT("$MYHOSTNAME");
+ case PROP_SYS_BOM:
+ return UCHAR_CONSTANT("$BOM");
default:
return UCHAR_CONSTANT("*invalid property id*");
}
}
-/* The following functions will support advanced output module
- * multithreading, once this is implemented. Currently, we
- * include them as hooks only. The idea is that we need to guard
- * some msg objects data fields against concurrent access if
- * we run on multiple threads. Please note that in any case this
- * is not necessary for calls from INPUT modules, because they
- * construct the message object and do this serially. Only when
- * the message is in the processing queue, multiple threads may
- * access a single object. Consequently, there are no guard functions
- * for "set" methods, as these are called during input. Only "get"
- * functions that modify important structures have them.
- * rgerhards, 2007-07-20
- * We now support locked and non-locked operations, depending on
- * the configuration of rsyslog. To support this, we use function
- * pointers. Initially, we start in non-locked mode. There, all
- * locking operations call into dummy functions. When locking is
- * enabled, the function pointers are changed to functions doing
- * actual work. We also introduced another MsgPrepareEnqueue() function
- * which initializes the locking structures, if needed. This is
- * necessary because internal messages during config file startup
- * processing are always created in non-locking mode. So we can
- * not initialize locking structures during constructions. We now
- * postpone this until when the message is fully constructed and
- * enqueued. Then we know the status of locking. This has a nice
- * side effect, and that is that during the initial creation of
- * the Msg object no locking needs to be done, which results in better
- * performance. -- rgerhards, 2008-01-05
- */
-static void (*funcLock)(msg_t *pMsg);
-static void (*funcUnlock)(msg_t *pMsg);
-static void (*funcDeleteMutex)(msg_t *pMsg);
-void (*funcMsgPrepareEnqueue)(msg_t *pMsg);
-#if 1 /* This is a debug aid */
-#define MsgLock(pMsg) funcLock(pMsg)
-#define MsgUnlock(pMsg) funcUnlock(pMsg)
-#else
-#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; }
-#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); }
-#endif
-
-/* the next function is a dummy to be used by the looking functions
- * when the class is not yet running in an environment where locking
- * is necessary. Please note that the need to lock can (and will) change
- * during a single run. Typically, this is depending on the operation mode
- * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05
- */
-static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg)
-{
- /* empty be design */
-}
-
-
-/* The following function prepares a message for enqueue into the queue. This is
- * where a message may be accessed by multiple threads. This implementation here
- * is the version for multiple concurrent acces. It initializes the locking
- * structures.
- * TODO: change to an iRet interface! -- rgerhards, 2008-07-14
- */
-static void MsgPrepareEnqueueLockingCase(msg_t *pThis)
-{
- BEGINfunc
- assert(pThis != NULL);
- pthread_mutex_init(&pThis->mut, NULL);
- pThis->bDoLock = 1;
- ENDfunc
-}
-
-
-/* ... and now the locking and unlocking implementations: */
-static void MsgLockLockingCase(msg_t *pThis)
-{
- /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */
- assert(pThis != NULL);
- if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */
- pthread_mutex_lock(&pThis->mut);
-}
-
-static void MsgUnlockLockingCase(msg_t *pThis)
-{
- /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */
- assert(pThis != NULL);
- if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */
- pthread_mutex_unlock(&pThis->mut);
-}
-
-/* delete the mutex object on message destruction (locking case)
- */
-static void MsgDeleteMutexLockingCase(msg_t *pThis)
-{
- assert(pThis != NULL);
- pthread_mutex_destroy(&pThis->mut);
-}
-
-/* enable multiple concurrent access on the message object
- * This works on a class-wide basis and can bot be undone.
- * That is, if it is once enabled, it can not be disabled during
- * the same run. When this function is called, no other thread
- * must manipulate message objects. Then we would have race conditions,
- * but guarding against this is counter-productive because it
- * would cost additional time. Plus, it would be a programming error.
- * rgerhards, 2008-01-05
- */
-rsRetVal MsgEnableThreadSafety(void)
-{
- DEFiRet;
- funcLock = MsgLockLockingCase;
- funcUnlock = MsgUnlockLockingCase;
- funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase;
- funcDeleteMutex = MsgDeleteMutexLockingCase;
- RETiRet;
-}
-
-/* end locking functions */
-
-
/* This is common code for all Constructors. It is defined in an
* inline'able function so that we can save a function call in the
* actual constructors (otherwise, the msgConstruct would need
@@ -626,13 +677,13 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
msg_t *pM;
assert(ppThis != NULL);
- CHKmalloc(pM = malloc(sizeof(msg_t)));
+ CHKmalloc(pM = MALLOC(sizeof(msg_t)));
objConstructSetObjInfo(pM); /* intialize object helper entities */
/* initialize members in ORDER they appear in structure (think "cache line"!) */
pM->flowCtlType = 0;
pM->bDoLock = 0;
- pM->bParseHOSTNAME = 0;
+ pM->bAlreadyFreed = 0;
pM->iRefCount = 1;
pM->iSeverity = -1;
pM->iFacility = -1;
@@ -661,7 +712,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
pM->pCSMSGID = NULL;
pM->pInputName = NULL;
pM->pRcvFromIP = NULL;
- pM->pRcvFrom = NULL;
+ pM->rcvFrom.pRcvFrom = NULL;
pM->pRuleset = NULL;
memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt));
memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP));
@@ -745,6 +796,9 @@ static inline void freeHOSTNAME(msg_t *pThis)
BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */
int currRefCount;
+# if HAVE_MALLOC_TRIM
+ int currCnt;
+# endif
CODESTARTobjDestruct(msg)
/* DEV Debugging only ! dbgprintf("msgDestruct\t0x%lx, Ref now: %d\n", (unsigned long)pThis, pThis->iRefCount - 1); */
# ifdef HAVE_ATOMIC_BUILTINS
@@ -756,14 +810,27 @@ CODESTARTobjDestruct(msg)
if(currRefCount == 0)
{
/* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */
+ /* The if below is included to try to nail down a well-hidden bug causing
+ * segfaults. I hope that do to the test code the problem is sooner detected and
+ * thus we get better data for debugging and resolving it. -- rgerhards, 2011-02-23.
+ * TODO: remove when no longer needed.
+ */
+ if(pThis->bAlreadyFreed)
+ abort();
+ pThis->bAlreadyFreed = 1;
+ /* end debug code */
if(pThis->pszRawMsg != pThis->szRawMsg)
free(pThis->pszRawMsg);
freeTAG(pThis);
freeHOSTNAME(pThis);
if(pThis->pInputName != NULL)
prop.Destruct(&pThis->pInputName);
- if(pThis->pRcvFrom != NULL)
- prop.Destruct(&pThis->pRcvFrom);
+ if((pThis->msgFlags & NEEDS_DNSRESOL) == 0) {
+ if(pThis->rcvFrom.pRcvFrom != NULL)
+ prop.Destruct(&pThis->rcvFrom.pRcvFrom);
+ } else {
+ free(pThis->rcvFrom.pfrominet);
+ }
if(pThis->pRcvFromIP != NULL)
prop.Destruct(&pThis->pRcvFromIP);
free(pThis->pszRcvdAt3164);
@@ -800,18 +867,10 @@ CODESTARTobjDestruct(msg)
* that we trim too often when the counter wraps.
*/
static unsigned iTrimCtr = 1;
-# ifdef HAVE_ATOMICS
- if(ATOMIC_INC_AND_FETCH(iTrimCtr) % 100000 == 0) {
- malloc_trim(128*1024);
- }
-# else
-static pthread_mutex_t mutTrimCtr = PTHREAD_MUTEX_INITIALIZER;
- d_pthread_mutex_lock(&mutTrimCtr);
- if(iTrimCtr++ % 100000 == 0) {
+ currCnt = ATOMIC_INC_AND_FETCH_unsigned(&iTrimCtr, &mutTrimCtr);
+ if(currCnt % 100000 == 0) {
malloc_trim(128*1024);
}
- d_pthread_mutex_unlock(&mutTrimCtr);
-# endif
}
# endif
} else {
@@ -858,6 +917,7 @@ ENDobjDestruct(msg)
msg_t* MsgDup(msg_t* pOld)
{
msg_t* pNew;
+ rsRetVal localRet;
assert(pOld != NULL);
@@ -870,7 +930,6 @@ msg_t* MsgDup(msg_t* pOld)
pNew->iRefCount = 1;
pNew->iSeverity = pOld->iSeverity;
pNew->iFacility = pOld->iFacility;
- pNew->bParseHOSTNAME = pOld->bParseHOSTNAME;
pNew->msgFlags = pOld->msgFlags;
pNew->iProtocolVersion = pOld->iProtocolVersion;
pNew->ttGenTime = pOld->ttGenTime;
@@ -879,9 +938,20 @@ msg_t* MsgDup(msg_t* pOld)
pNew->iLenMSG = pOld->iLenMSG;
pNew->iLenTAG = pOld->iLenTAG;
pNew->iLenHOSTNAME = pOld->iLenHOSTNAME;
- if(pOld->pRcvFrom != NULL) {
- pNew->pRcvFrom = pOld->pRcvFrom;
- prop.AddRef(pNew->pRcvFrom);
+ if((pOld->msgFlags & NEEDS_DNSRESOL)) {
+ localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet);
+ if(localRet != RS_RET_OK) {
+ /* if something fails, we accept loss of this property, it is
+ * better than losing the whole message.
+ */
+ pNew->msgFlags &= ~NEEDS_DNSRESOL;
+ pNew->rcvFrom.pRcvFrom = NULL; /* make sure no dangling values */
+ }
+ } else {
+ if(pOld->rcvFrom.pRcvFrom != NULL) {
+ pNew->rcvFrom.pRcvFrom = pOld->rcvFrom.pRcvFrom;
+ prop.AddRef(pNew->rcvFrom.pRcvFrom);
+ }
}
if(pOld->pRcvFromIP != NULL) {
pNew->pRcvFromIP = pOld->pRcvFromIP;
@@ -944,7 +1014,7 @@ msg_t* MsgDup(msg_t* pOld)
* We do not serialize the cache properties. We re-create them when needed.
* This saves us a lot of memory. Performance is no concern, as serializing
* is a so slow operation that recration of the caches does not count. Also,
- * we do not serialize bParseHOSTNAME, as this is only a helper variable
+ * we do not serialize --currently none--, as this is only a helper variable
* during msg construction - and never again used later.
* rgerhards, 2008-01-03
*/
@@ -1145,15 +1215,21 @@ char *getProtocolVersionString(msg_t *pM)
}
-static char *getRawMsg(msg_t *pM)
+static inline void
+getRawMsg(msg_t *pM, uchar **pBuf, int *piLen)
{
- if(pM == NULL)
- return "";
- else
- if(pM->pszRawMsg == NULL)
- return "";
- else
- return (char*)pM->pszRawMsg;
+ if(pM == NULL) {
+ *pBuf= UCHAR_CONSTANT("");
+ *piLen = 0;
+ } else {
+ if(pM->pszRawMsg == NULL) {
+ *pBuf= UCHAR_CONSTANT("");
+ *piLen = 0;
+ } else {
+ *pBuf = pM->pszRawMsg;
+ *piLen = pM->iLenRawMsg;
+ }
+ }
}
@@ -1198,7 +1274,8 @@ static int getPRIi(msg_t *pM)
/* Get PRI value in text form
*/
-static inline char *getPRI(msg_t *pM)
+char *
+getPRI(msg_t *pM)
{
/* PRI is a number in the range 0..191. Thus, we use a simple lookup table to obtain the
* string value. It looks a bit clumpsy here in code ;)
@@ -1213,7 +1290,8 @@ static inline char *getPRI(msg_t *pM)
}
-static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt)
+char *
+getTimeReported(msg_t *pM, enum tplFormatTypes eFmt)
{
BEGINfunc
if(pM == NULL)
@@ -1234,7 +1312,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt)
case tplFmtMySQLDate:
MsgLock(pM);
if(pM->pszTIMESTAMP_MySQL == NULL) {
- if((pM->pszTIMESTAMP_MySQL = malloc(15)) == NULL) {
+ if((pM->pszTIMESTAMP_MySQL = MALLOC(15)) == NULL) {
MsgUnlock(pM);
return "";
}
@@ -1245,7 +1323,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt)
case tplFmtPgSQLDate:
MsgLock(pM);
if(pM->pszTIMESTAMP_PgSQL == NULL) {
- if((pM->pszTIMESTAMP_PgSQL = malloc(21)) == NULL) {
+ if((pM->pszTIMESTAMP_PgSQL = MALLOC(21)) == NULL) {
MsgUnlock(pM);
return "";
}
@@ -1286,7 +1364,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt)
case tplFmtDefault:
MsgLock(pM);
if(pM->pszRcvdAt3164 == NULL) {
- if((pM->pszRcvdAt3164 = malloc(16)) == NULL) {
+ if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) {
MsgUnlock(pM);
return "";
}
@@ -1297,7 +1375,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt)
case tplFmtMySQLDate:
MsgLock(pM);
if(pM->pszRcvdAt_MySQL == NULL) {
- if((pM->pszRcvdAt_MySQL = malloc(15)) == NULL) {
+ if((pM->pszRcvdAt_MySQL = MALLOC(15)) == NULL) {
MsgUnlock(pM);
return "";
}
@@ -1308,7 +1386,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt)
case tplFmtPgSQLDate:
MsgLock(pM);
if(pM->pszRcvdAt_PgSQL == NULL) {
- if((pM->pszRcvdAt_PgSQL = malloc(21)) == NULL) {
+ if((pM->pszRcvdAt_PgSQL = MALLOC(21)) == NULL) {
MsgUnlock(pM);
return "";
}
@@ -1320,7 +1398,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt)
case tplFmtRFC3164BuggyDate:
MsgLock(pM);
if(pM->pszRcvdAt3164 == NULL) {
- if((pM->pszRcvdAt3164 = malloc(16)) == NULL) {
+ if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) {
MsgUnlock(pM);
return "";
}
@@ -1332,7 +1410,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt)
case tplFmtRFC3339Date:
MsgLock(pM);
if(pM->pszRcvdAt3339 == NULL) {
- if((pM->pszRcvdAt3339 = malloc(33)) == NULL) {
+ if((pM->pszRcvdAt3339 = MALLOC(33)) == NULL) {
MsgUnlock(pM);
return "";
}
@@ -1497,7 +1575,7 @@ finalize_it:
* This must be called WITHOUT the message lock being held.
* rgerhards, 2009-06-26
*/
-static inline void preparePROCID(msg_t *pM, bool bLockMutex)
+static inline void preparePROCID(msg_t *pM, sbool bLockMutex)
{
if(pM->pCSPROCID == NULL) {
if(bLockMutex == LOCK_MUTEX)
@@ -1514,7 +1592,7 @@ static inline void preparePROCID(msg_t *pM, bool bLockMutex)
#if 0
/* rgerhards, 2005-11-24
*/
-static inline int getPROCIDLen(msg_t *pM, bool bLockMutex)
+static inline int getPROCIDLen(msg_t *pM, sbool bLockMutex)
{
assert(pM != NULL);
preparePROCID(pM, bLockMutex);
@@ -1525,11 +1603,21 @@ static inline int getPROCIDLen(msg_t *pM, bool bLockMutex)
/* rgerhards, 2005-11-24
*/
-char *getPROCID(msg_t *pM, bool bLockMutex)
+char *getPROCID(msg_t *pM, sbool bLockMutex)
{
+ uchar *pszRet;
+
ISOBJ_TYPE_assert(pM, msg);
- preparePROCID(pM, bLockMutex);
- return (pM->pCSPROCID == NULL) ? "-" : (char*) cstrGetSzStrNoNULL(pM->pCSPROCID);
+ if(bLockMutex == LOCK_MUTEX)
+ MsgUnlock(pM);
+ preparePROCID(pM, MUTEX_ALREADY_LOCKED);
+ if(pM->pCSPROCID == NULL)
+ pszRet = UCHAR_CONSTANT("");
+ else
+ pszRet = rsCStrGetSzStrNoNULL(pM->pCSPROCID);
+ if(bLockMutex == LOCK_MUTEX)
+ MsgUnlock(pM);
+ return (char*) pszRet;
}
@@ -1551,14 +1639,21 @@ finalize_it:
}
-/* rgerhards, 2005-11-24
+/* al, 2011-07-26: LockMsg to avoid race conditions
*/
static inline char *getMSGID(msg_t *pM)
{
- return (pM->pCSMSGID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSMSGID);
+ if (pM->pCSMSGID == NULL) {
+ return "-";
+ }
+ else {
+ MsgLock(pM);
+ char* pszreturn = (char*) rsCStrGetSzStrNoNULL(pM->pCSMSGID);
+ MsgUnlock(pM);
+ return pszreturn;
+ }
}
-
/* rgerhards 2009-06-12: set associated ruleset
*/
void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset)
@@ -1576,6 +1671,8 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf)
uchar *pBuf;
assert(pMsg != NULL);
+dbgprintf("MsgSetTAG in: len %d, pszBuf: %s\n", lenBuf, pszBuf);
+
freeTAG(pMsg);
pMsg->iLenTAG = lenBuf;
@@ -1583,7 +1680,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf)
/* small enough: use fixed buffer (faster!) */
pBuf = pMsg->TAG.szBuf;
} else {
- if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) {
+ if((pBuf = (uchar*) MALLOC(pMsg->iLenTAG + 1)) == NULL) {
/* truncate message, better than completely loosing it... */
pBuf = pMsg->TAG.szBuf;
pMsg->iLenTAG = CONF_TAG_BUFSIZE - 1;
@@ -1594,6 +1691,8 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf)
memcpy(pBuf, pszBuf, pMsg->iLenTAG);
pBuf[pMsg->iLenTAG] = '\0'; /* this also works with truncation! */
+
+dbgprintf("MsgSetTAG exit: pMsg->iLenTAG %d, pMsg->TAG.szBuf: %s\n", pMsg->iLenTAG, pMsg->TAG.szBuf);
}
@@ -1604,7 +1703,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf)
* if there is a TAG and, if not, if it can emulate it.
* rgerhards, 2005-11-24
*/
-static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex)
+static inline void tryEmulateTAG(msg_t *pM, sbool bLockMutex)
{
size_t lenTAG;
uchar bufTAG[CONF_TAG_MAXSIZE];
@@ -1612,8 +1711,11 @@ static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex)
if(bLockMutex == LOCK_MUTEX)
MsgLock(pM);
- if(pM->iLenTAG > 0)
+ if(pM->iLenTAG > 0) {
+ if(bLockMutex == LOCK_MUTEX)
+ MsgUnlock(pM);
return; /* done, no need to emulate */
+ }
if(getProtocolVersion(pM) == 1) {
if(!strcmp(getPROCID(pM, MUTEX_ALREADY_LOCKED), "-")) {
@@ -1623,7 +1725,7 @@ static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex)
/* now we can try to emulate */
lenTAG = snprintf((char*)bufTAG, CONF_TAG_MAXSIZE, "%s[%s]",
getAPPNAME(pM, MUTEX_ALREADY_LOCKED), getPROCID(pM, MUTEX_ALREADY_LOCKED));
- bufTAG[32] = '\0'; /* just to make sure... */
+ bufTAG[sizeof(bufTAG)-1] = '\0'; /* just to make sure... */
MsgSetTAG(pM, bufTAG, lenTAG);
}
}
@@ -1632,7 +1734,7 @@ static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex)
}
-static inline void
+void
getTAG(msg_t *pM, uchar **ppBuf, int *piLen)
{
if(pM == NULL) {
@@ -1657,12 +1759,13 @@ int getHOSTNAMELen(msg_t *pM)
if(pM == NULL)
return 0;
else
- if(pM->pszHOSTNAME == NULL)
- if(pM->pRcvFrom == NULL)
+ if(pM->pszHOSTNAME == NULL) {
+ resolveDNS(pM);
+ if(pM->rcvFrom.pRcvFrom == NULL)
return 0;
else
- return prop.GetStringLen(pM->pRcvFrom);
- else
+ return prop.GetStringLen(pM->rcvFrom.pRcvFrom);
+ } else
return pM->iLenHOSTNAME;
}
@@ -1673,12 +1776,13 @@ char *getHOSTNAME(msg_t *pM)
return "";
else
if(pM->pszHOSTNAME == NULL) {
- if(pM->pRcvFrom == NULL) {
+ resolveDNS(pM);
+ if(pM->rcvFrom.pRcvFrom == NULL) {
return "";
} else {
uchar *psz;
int len;
- prop.GetString(pM->pRcvFrom, &psz, &len);
+ prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len);
return (char*) psz;
}
} else {
@@ -1692,13 +1796,15 @@ uchar *getRcvFrom(msg_t *pM)
uchar *psz;
int len;
BEGINfunc
+
if(pM == NULL) {
psz = UCHAR_CONSTANT("");
} else {
- if(pM->pRcvFrom == NULL)
+ resolveDNS(pM);
+ if(pM->rcvFrom.pRcvFrom == NULL)
psz = UCHAR_CONSTANT("");
else
- prop.GetString(pM->pRcvFrom, &psz, &len);
+ prop.GetString(pM->rcvFrom.pRcvFrom, &psz, &len);
}
ENDfunc
return psz;
@@ -1738,14 +1844,22 @@ static int getStructuredDataLen(msg_t *pM)
*/
static inline char *getStructuredData(msg_t *pM)
{
- return (pM->pCSStrucData == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSStrucData);
+ uchar *pszRet;
+
+ MsgUnlock(pM);
+ if(pM->pCSStrucData == NULL)
+ pszRet = UCHAR_CONSTANT("-");
+ else
+ pszRet = rsCStrGetSzStrNoNULL(pM->pCSStrucData);
+ MsgUnlock(pM);
+ return (char*) pszRet;
}
/* check if we have a ProgramName, and, if not, try to aquire/emulate it.
* rgerhards, 2009-06-26
*/
-static inline void prepareProgramName(msg_t *pM, bool bLockMutex)
+static inline void prepareProgramName(msg_t *pM, sbool bLockMutex)
{
if(pM->pCSProgName == NULL) {
if(bLockMutex == LOCK_MUTEX)
@@ -1764,7 +1878,7 @@ static inline void prepareProgramName(msg_t *pM, bool bLockMutex)
/* get the length of the "programname" sz string
* rgerhards, 2005-10-19
*/
-int getProgramNameLen(msg_t *pM, bool bLockMutex)
+int getProgramNameLen(msg_t *pM, sbool bLockMutex)
{
assert(pM != NULL);
prepareProgramName(pM, bLockMutex);
@@ -1775,10 +1889,20 @@ int getProgramNameLen(msg_t *pM, bool bLockMutex)
/* get the "programname" as sz string
* rgerhards, 2005-10-19
*/
-char *getProgramName(msg_t *pM, bool bLockMutex)
+uchar *getProgramName(msg_t *pM, sbool bLockMutex)
{
- prepareProgramName(pM, bLockMutex);
- return (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName);
+ uchar *pszRet;
+
+ if(bLockMutex == LOCK_MUTEX)
+ MsgUnlock(pM);
+ prepareProgramName(pM, MUTEX_ALREADY_LOCKED);
+ if(pM->pCSProgName == NULL)
+ pszRet = UCHAR_CONSTANT("");
+ else
+ pszRet = rsCStrGetSzStrNoNULL(pM->pCSProgName);
+ if(bLockMutex == LOCK_MUTEX)
+ MsgUnlock(pM);
+ return pszRet;
}
@@ -1795,7 +1919,7 @@ static void tryEmulateAPPNAME(msg_t *pM)
if(getProtocolVersion(pM) == 0) {
/* only then it makes sense to emulate */
- MsgSetAPPNAME(pM, getProgramName(pM, MUTEX_ALREADY_LOCKED));
+ MsgSetAPPNAME(pM, (char*)getProgramName(pM, MUTEX_ALREADY_LOCKED));
}
}
@@ -1805,7 +1929,7 @@ static void tryEmulateAPPNAME(msg_t *pM)
* This must be called WITHOUT the message lock being held.
* rgerhards, 2009-06-26
*/
-static inline void prepareAPPNAME(msg_t *pM, bool bLockMutex)
+static inline void prepareAPPNAME(msg_t *pM, sbool bLockMutex)
{
if(pM->pCSAPPNAME == NULL) {
if(bLockMutex == LOCK_MUTEX)
@@ -1822,16 +1946,26 @@ static inline void prepareAPPNAME(msg_t *pM, bool bLockMutex)
/* rgerhards, 2005-11-24
*/
-char *getAPPNAME(msg_t *pM, bool bLockMutex)
+char *getAPPNAME(msg_t *pM, sbool bLockMutex)
{
+ uchar *pszRet;
+
assert(pM != NULL);
- prepareAPPNAME(pM, bLockMutex);
- return (pM->pCSAPPNAME == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSAPPNAME);
+ if(bLockMutex == LOCK_MUTEX)
+ MsgUnlock(pM);
+ prepareAPPNAME(pM, MUTEX_ALREADY_LOCKED);
+ if(pM->pCSAPPNAME == NULL)
+ pszRet = UCHAR_CONSTANT("");
+ else
+ pszRet = rsCStrGetSzStrNoNULL(pM->pCSAPPNAME);
+ if(bLockMutex == LOCK_MUTEX)
+ MsgUnlock(pM);
+ return (char*)pszRet;
}
/* rgerhards, 2005-11-24
*/
-static int getAPPNAMELen(msg_t *pM, bool bLockMutex)
+static int getAPPNAMELen(msg_t *pM, sbool bLockMutex)
{
assert(pM != NULL);
prepareAPPNAME(pM, bLockMutex);
@@ -1854,6 +1988,28 @@ void MsgSetInputName(msg_t *pThis, prop_t *inputName)
}
+/* Set the pfrominet socket store, so that we can obtain the peer at some
+ * later time. Note that we do not check if pRcvFrom is already set, so this
+ * function must only be called during message creation.
+ * NOTE: msgFlags is NOT set. While this is somewhat a violation of layers,
+ * it is done because it gains us some performance. So the caller must make
+ * sure the message flags are properly maintained. For all current callers,
+ * this is always the case and without extra effort required.
+ * rgerhards, 2009-11-17
+ */
+rsRetVal
+msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa){
+ DEFiRet;
+ assert(pThis->rcvFrom.pRcvFrom == NULL);
+
+ CHKmalloc(pThis->rcvFrom.pfrominet = malloc(sizeof(struct sockaddr_storage)));
+ memcpy(pThis->rcvFrom.pfrominet, sa, sizeof(struct sockaddr_storage));
+
+finalize_it:
+ RETiRet;
+}
+
+
/* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef()
* on the property, because this must be done in all current cases and there
* is no case expected where this may not be necessary.
@@ -1864,9 +2020,15 @@ void MsgSetRcvFrom(msg_t *pThis, prop_t *new)
assert(pThis != NULL);
prop.AddRef(new);
- if(pThis->pRcvFrom != NULL)
- prop.Destruct(&pThis->pRcvFrom);
- pThis->pRcvFrom = new;
+ if(pThis->msgFlags & NEEDS_DNSRESOL) {
+ if(pThis->rcvFrom.pfrominet != NULL)
+ free(pThis->rcvFrom.pfrominet);
+ pThis->msgFlags &= ~NEEDS_DNSRESOL;
+ } else {
+ if(pThis->rcvFrom.pRcvFrom != NULL)
+ prop.Destruct(&pThis->rcvFrom.pRcvFrom);
+ }
+ pThis->rcvFrom.pRcvFrom = new;
}
@@ -1948,7 +2110,7 @@ void MsgSetHOSTNAME(msg_t *pThis, uchar* pszHOSTNAME, int lenHOSTNAME)
if(pThis->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) {
/* small enough: use fixed buffer (faster!) */
pThis->pszHOSTNAME = pThis->szHOSTNAME;
- } else if((pThis->pszHOSTNAME = (uchar*) malloc(pThis->iLenHOSTNAME + 1)) == NULL) {
+ } else if((pThis->pszHOSTNAME = (uchar*) MALLOC(pThis->iLenHOSTNAME + 1)) == NULL) {
/* truncate message, better than completely loosing it... */
pThis->pszHOSTNAME = pThis->szHOSTNAME;
pThis->iLenHOSTNAME = CONF_HOSTNAME_BUFSIZE - 1;
@@ -2000,7 +2162,7 @@ rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG)
lenNew = pThis->iLenRawMsg + lenMSG - pThis->iLenMSG;
if(lenMSG > pThis->iLenMSG && lenNew >= CONF_RAWMSG_BUFSIZE) {
/* we have lost our "bet" and need to alloc a new buffer ;) */
- CHKmalloc(bufNew = malloc(lenNew + 1));
+ CHKmalloc(bufNew = MALLOC(lenNew + 1));
memcpy(bufNew, pThis->pszRawMsg, pThis->offMSG);
if(pThis->pszRawMsg != pThis->szRawMsg)
free(pThis->pszRawMsg);
@@ -2033,7 +2195,7 @@ void MsgSetRawMsg(msg_t *pThis, char* pszRawMsg, size_t lenMsg)
if(pThis->iLenRawMsg < CONF_RAWMSG_BUFSIZE) {
/* small enough: use fixed buffer (faster!) */
pThis->pszRawMsg = pThis->szRawMsg;
- } else if((pThis->pszRawMsg = (uchar*) malloc(pThis->iLenRawMsg + 1)) == NULL) {
+ } else if((pThis->pszRawMsg = (uchar*) MALLOC(pThis->iLenRawMsg + 1)) == NULL) {
/* truncate message, better than completely loosing it... */
pThis->pszRawMsg = pThis->szRawMsg;
pThis->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1;
@@ -2089,7 +2251,7 @@ static uchar *getNOW(eNOWType eNow)
uchar *pBuf;
struct syslogTime t;
- if((pBuf = (uchar*) malloc(sizeof(uchar) * tmpBUFSIZE)) == NULL) {
+ if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) {
return NULL;
}
@@ -2200,12 +2362,13 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
break;
case PROP_HOSTNAME:
pRes = (uchar*)getHOSTNAME(pMsg);
+ bufLen = getHOSTNAMELen(pMsg);
break;
case PROP_SYSLOGTAG:
getTAG(pMsg, &pRes, &bufLen);
break;
case PROP_RAWMSG:
- pRes = (uchar*)getRawMsg(pMsg);
+ getRawMsg(pMsg, &pRes, &bufLen);
break;
/* enable this, if someone actually uses UxTradMsg, delete after some time has
* passed and nobody complained -- rgerhards, 2009-06-16
@@ -2226,7 +2389,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
pRes = (uchar*)getPRI(pMsg);
break;
case PROP_PRI_TEXT:
- pBuf = malloc(20 * sizeof(uchar));
+ pBuf = MALLOC(20 * sizeof(uchar));
if(pBuf == NULL) {
RET_OUT_OF_MEMORY;
} else {
@@ -2236,6 +2399,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
break;
case PROP_IUT:
pRes = UCHAR_CONSTANT("1"); /* always 1 for syslog messages (a MonitorWare thing;)) */
+ bufLen = 1;
break;
case PROP_SYSLOGFACILITY:
pRes = (uchar*)getFacility(pMsg);
@@ -2253,7 +2417,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
pRes = (uchar*)getTimeGenerated(pMsg, pTpe->data.field.eDateFormat);
break;
case PROP_PROGRAMNAME:
- pRes = (uchar*)getProgramName(pMsg, LOCK_MUTEX);
+ pRes = getProgramName(pMsg, LOCK_MUTEX);
break;
case PROP_PROTOCOL_VERSION:
pRes = (uchar*)getProtocolVersionString(pMsg);
@@ -2321,6 +2485,12 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
case PROP_SYS_MYHOSTNAME:
pRes = glbl.GetLocalHostName();
break;
+ case PROP_SYS_BOM:
+ if(*pbMustBeFreed == 1)
+ free(pRes);
+ pRes = (uchar*) "\xEF\xBB\xBF";
+ *pbMustBeFreed = 0;
+ break;
default:
/* there is no point in continuing, we may even otherwise render the
* error message unreadable. rgerhards, 2007-07-10
@@ -2381,7 +2551,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
/* we got our end pointer, now do the copy */
/* TODO: code copied from below, this is a candidate for a separate function */
iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */
- pBufStart = pBuf = malloc((iLen + 1) * sizeof(char));
+ pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char));
if(pBuf == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2427,7 +2597,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
; /*DO NOTHING*/
} else {
iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */
- pBufStart = pBuf = malloc((iLen + 1) * sizeof(char));
+ pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char));
if(pBuf == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2547,7 +2717,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
iLenBuf = pmatch[pTpe->data.field.iSubMatchToUse].rm_eo
- pmatch[pTpe->data.field.iSubMatchToUse].rm_so;
- pB = malloc((iLenBuf + 1) * sizeof(uchar));
+ pB = MALLOC((iLenBuf + 1) * sizeof(uchar));
if (pB == NULL) {
if (*pbMustBeFreed == 1)
@@ -2604,7 +2774,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
uchar *pBStart;
uchar *pB;
uchar *pSrc;
- pBStart = pB = malloc((bufLen + 1) * sizeof(char));
+ pBStart = pB = MALLOC((bufLen + 1) * sizeof(char));
if(pB == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2650,7 +2820,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
}
if(bDropped) {
- pDst = pDstStart = malloc(iLenBuf + 1);
+ pDst = pDstStart = MALLOC(iLenBuf + 1);
if(pDst == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2685,7 +2855,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
} else {
if(bufLen == -1)
bufLen = ustrlen(pRes);
- pDst = pDstStart = malloc(bufLen + 1);
+ pDst = pDstStart = MALLOC(bufLen + 1);
if(pDst == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2724,7 +2894,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
int i;
iLenBuf += iNumCC * 4;
- pBStart = pB = malloc((iLenBuf + 1) * sizeof(uchar));
+ pBStart = pB = MALLOC((iLenBuf + 1) * sizeof(uchar));
if(pB == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2750,6 +2920,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
}
}
+dbgprintf("prop repl 4, pRes='%s', len %d\n", pRes, bufLen);
/* Take care of spurious characters to make the property safe
* for a path definition
*/
@@ -2769,7 +2940,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
}
if(bDropped) {
- pDst = pDstStart = malloc(iLenBuf + 1);
+ pDst = pDstStart = MALLOC(iLenBuf + 1);
if(pDst == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2804,7 +2975,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
} else {
if(bufLen == -1)
bufLen = ustrlen(pRes);
- pDst = pDstStart = malloc(bufLen + 1);
+ pDst = pDstStart = MALLOC(bufLen + 1);
if(pDst == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2860,7 +3031,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
/* check if we need to obtain a private copy */
if(*pbMustBeFreed == 0) {
/* ok, original copy, need a private one */
- pB = malloc((iLn + 1) * sizeof(uchar));
+ pB = MALLOC((iLn + 1) * sizeof(uchar));
if(pB == NULL) {
RET_OUT_OF_MEMORY;
}
@@ -2888,7 +3059,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
bufLen = ustrlen(pRes);
iBufLen = bufLen;
/* the malloc may be optimized, we currently use the worst case... */
- pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(uchar));
+ pBStart = pDst = MALLOC((2 * iBufLen + 3) * sizeof(uchar));
if(pDst == NULL) {
if(*pbMustBeFreed == 1)
free(pRes);
@@ -2914,6 +3085,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
bufLen = ustrlen(pRes);
*pPropLen = bufLen;
+dbgprintf("end prop repl, pRes='%s', len %d\n", pRes, bufLen);
ENDfunc
return(pRes);
}
@@ -3088,6 +3260,10 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE)
funcUnlock = MsgLockingDummy;
funcDeleteMutex = MsgLockingDummy;
funcMsgPrepareEnqueue = MsgLockingDummy;
+ /* some more inits */
+# if HAVE_MALLOC_TRIM
+ INIT_ATOMIC_HELPER_MUT(mutTrimCtr);
+# endif
ENDObjClassInit(msg)
/* vim:set ai:
*/
diff --git a/runtime/msg.h b/runtime/msg.h
index a6923d52..26a07aca 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -61,14 +61,8 @@ struct msg {
once data has entered the queue, this property is no longer needed. */
pthread_mutex_t mut;
int iRefCount; /* reference counter (0 = unused) */
- bool bDoLock; /* use the mutex? */
- bool bParseHOSTNAME; /* should the hostname be parsed from the message? */
- /* background: the hostname is not present on "regular" messages
- * received via UNIX domain sockets from the same machine. However,
- * it is available when we have a forwarder (e.g. rfc3195d) using local
- * sockets. All in all, the parser would need parse templates, that would
- * resolve all these issues... rgerhards, 2005-10-06
- */
+ sbool bDoLock; /* use the mutex? */
+ sbool bAlreadyFreed; /* aid to help detect a well-hidden bad bug -- TODO: remove when no longer needed */
short iSeverity; /* the severity 0..7 */
short iFacility; /* Facility code 0 .. 23*/
short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */
@@ -96,8 +90,12 @@ struct msg {
cstr_t *pCSPROCID; /* PROCID */
cstr_t *pCSMSGID; /* MSGID */
prop_t *pInputName; /* input name property */
- prop_t *pRcvFrom; /* name of system message was received from */
prop_t *pRcvFromIP; /* IP of system message was received from */
+ union {
+ prop_t *pRcvFrom;/* name of system message was received from */
+ struct sockaddr_storage *pfrominet; /* unresolved name */
+ } rcvFrom;
+
ruleset_t *pRuleset; /* ruleset to be used for processing this message */
time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp.
While this field looks redundant, it is required because a Unix timestamp
@@ -115,8 +113,8 @@ struct msg {
uchar *pszTAG; /* pointer to tag value */
uchar szBuf[CONF_TAG_BUFSIZE];
} TAG;
- char pszTimestamp3164[16];
- char pszTimestamp3339[33];
+ char pszTimestamp3164[CONST_LEN_TIMESTAMP_3164 + 1];
+ char pszTimestamp3339[CONST_LEN_TIMESTAMP_3339 + 1];
char pszTIMESTAMP_SecFrac[7]; /* Note: a pointer is 64 bits/8 char, so this is actually fewer than a pointer! */
char pszRcvdAt_SecFrac[7]; /* same as above. Both are fractional seconds for their respective timestamp */
};
@@ -131,6 +129,8 @@ struct msg {
#define MARK 0x008 /* this message is a mark */
#define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */
#define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */
+#define NEEDS_DNSRESOL 0x040 /* fromhost address is unresolved and must be locked up via DNS reverse lookup first */
+#define NEEDS_ACLCHK_U 0x080 /* check UDP ACLs after DNS resolution has been done in main queue consumer */
#define NO_PRI_IN_RAW 0x100 /* rawmsg does not include a PRI (Solaris!), but PRI is already set correctly in the msg object */
@@ -151,6 +151,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf);
void MsgSetRuleset(msg_t *pMsg, ruleset_t*);
rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl);
rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData);
+rsRetVal msgSetFromSockinfo(msg_t *pThis, struct sockaddr_storage *sa);
void MsgSetRcvFrom(msg_t *pMsg, prop_t*);
void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **);
rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*);
@@ -167,19 +168,22 @@ char *textpri(char *pRes, size_t pResLen, int pri);
rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar);
rsRetVal MsgEnableThreadSafety(void);
uchar *getRcvFrom(msg_t *pM);
+void getTAG(msg_t *pM, uchar **ppBuf, int *piLen);
+char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt);
+char *getPRI(msg_t *pMsg);
/* TODO: remove these five (so far used in action.c) */
uchar *getMSG(msg_t *pM);
char *getHOSTNAME(msg_t *pM);
-char *getPROCID(msg_t *pM, bool bLockMutex);
-char *getAPPNAME(msg_t *pM, bool bLockMutex);
+char *getPROCID(msg_t *pM, sbool bLockMutex);
+char *getAPPNAME(msg_t *pM, sbool bLockMutex);
int getMSGLen(msg_t *pM);
char *getHOSTNAME(msg_t *pM);
int getHOSTNAMELen(msg_t *pM);
-char *getProgramName(msg_t *pM, bool bLockMutex);
-int getProgramNameLen(msg_t *pM, bool bLockMutex);
+uchar *getProgramName(msg_t *pM, sbool bLockMutex);
+int getProgramNameLen(msg_t *pM, sbool bLockMutex);
uchar *getRcvFrom(msg_t *pM);
rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID);
uchar *propIDToName(propid_t propID);
@@ -211,6 +215,16 @@ MsgSetRawMsgSize(msg_t *pMsg, size_t newLen)
}
+/* get the ruleset that is associated with the ruleset.
+ * May be NULL. -- rgerhards, 2009-10-27
+ */
+static inline ruleset_t*
+MsgGetRuleset(msg_t *pMsg)
+{
+ return pMsg->pRuleset;
+}
+
+
#endif /* #ifndef MSG_H_INCLUDED */
/* vim:set ai:
*/
diff --git a/runtime/net.c b/runtime/net.c
index 8fd8cc2b..4781739f 100644
--- a/runtime/net.c
+++ b/runtime/net.c
@@ -69,6 +69,7 @@
#endif
MODULE_TYPE_LIB
+MODULE_TYPE_NOKEEP
/* static data */
DEFobjStaticHelpers
@@ -173,7 +174,7 @@ AddPermittedPeerWildcard(permittedPeers_t *pPeer, uchar* pszStr, size_t lenStr)
/* alloc memory for the domain component. We may waste a byte or
* two, but that's ok.
*/
- CHKmalloc(pNew->pszDomainPart = malloc(lenStr +1 ));
+ CHKmalloc(pNew->pszDomainPart = MALLOC(lenStr +1 ));
}
if(pszStr[0] == '*') {
@@ -695,7 +696,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS
case AF_INET: /* add IPv4 */
iSignificantBits = 32;
allowIP.flags = 0;
- if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) {
+ if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen);
@@ -710,7 +711,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS
iSignificantBits = 32;
allowIP.flags = 0;
- if((allowIP.addr.NetAddr = malloc(sizeof(struct sockaddr_in)))
+ if((allowIP.addr.NetAddr = MALLOC(sizeof(struct sockaddr_in)))
== NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
@@ -732,7 +733,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS
iSignificantBits = 128;
allowIP.flags = 0;
- if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) {
+ if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen);
@@ -892,15 +893,18 @@ rsRetVal addAllowedSenderLine(char* pName, uchar** ppRestOfConfLine)
* including IPv4/v6 as well as domain name wildcards.
* This is a helper to isAllowedSender. As it is only called once, it is
* declared inline.
- * Returns 0 if they do not match, something else otherwise.
- * contributed 1007-07-16 by mildew@gmail.com
+ * Returns 0 if they do not match, 1 if they match and 2 if a DNS name would have been required.
+ * contributed 2007-07-16 by mildew@gmail.com
*/
-static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost)
+static inline int
+MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS)
{
assert(pAllow != NULL);
assert(pFrom != NULL);
if(F_ISSET(pAllow->flags, ADDR_NAME)) {
+ if(bChkDNS == 0)
+ return 2;
dbgprintf("MaskCmp: host=\"%s\"; pattern=\"%s\"\n", pszFromHost, pAllow->addr.HostWildcard);
# if !defined(FNM_CASEFOLD)
@@ -967,18 +971,22 @@ static inline int MaskCmp(struct NetAddr *pAllow, uint8_t bits, struct sockaddr
/* check if a sender is allowed. The root of the the allowed sender.
* list must be proveded by the caller. As such, this function can be
* used to check both UDP and TCP allowed sender lists.
- * returns 1, if the sender is allowed, 0 otherwise.
+ * returns 1, if the sender is allowed, 0 if not and 2 if we could not
+ * obtain a result because we would need a dns name, which we don't have
+ * (2 was added rgerhards, 2009-11-16).
* rgerhards, 2005-09-26
*/
-static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost)
+static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS)
{
struct AllowedSenders *pAllow;
struct AllowedSenders *pAllowRoot;
+ int bNeededDNS = 0; /* partial check because we could not resolve DNS? */
+ int ret;
assert(pFrom != NULL);
if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK)
- return 0; /* if something went wrong, we denie access - that's the better choice... */
+ return 0; /* if something went wrong, we deny access - that's the better choice... */
if(pAllowRoot == NULL)
return 1; /* checking disabled, everything is valid! */
@@ -990,10 +998,20 @@ static int isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *p
* that the sender is disallowed.
*/
for(pAllow = pAllowRoot ; pAllow != NULL ; pAllow = pAllow->pNext) {
- if (MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost))
+ ret = MaskCmp (&(pAllow->allowedSender), pAllow->SignificantBits, pFrom, pszFromHost, bChkDNS);
+ if(ret == 1)
return 1;
+ else if(ret == 2)
+ bNeededDNS = 2;
}
- return 0;
+ return bNeededDNS;
+}
+
+
+/* legacy API, not to be used any longer */
+static int
+isAllowedSender(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost) {
+ return isAllowedSender2(pszType, pFrom, pszFromHost, 1);
}
@@ -1306,7 +1324,7 @@ getLocalHostname(uchar **ppName)
do {
if(buf == NULL) {
buf_len = 128; /* Initial guess */
- CHKmalloc(buf = malloc(buf_len));
+ CHKmalloc(buf = MALLOC(buf_len));
} else {
buf_len += buf_len;
CHKmalloc(buf = realloc (buf, buf_len));
@@ -1370,7 +1388,7 @@ int *create_udp_socket(uchar *hostname, uchar *pszPort, int bIsServer)
/* Count max number of sockets we may open */
for (maxs = 0, r = res; r != NULL ; r = r->ai_next, maxs++)
/* EMPTY */;
- socks = malloc((maxs+1) * sizeof(int));
+ socks = MALLOC((maxs+1) * sizeof(int));
if (socks == NULL) {
errmsg.LogError(0, NO_ERRCODE, "couldn't allocate memory for UDP sockets, suspending UDP message reception");
freeaddrinfo(res);
@@ -1533,12 +1551,36 @@ static int CmpHost(struct sockaddr_storage *s1, struct sockaddr_storage* s2, siz
ret = memcmp(s1, s2, socklen);
}
-dbgprintf("CmpHost returns %d\n", ret);
finalize_it:
return ret;
}
+
+/* check if restrictions (ALCs) exists. The goal of this function is to disable the
+ * somewhat time-consuming ACL checks if no restrictions are defined (the usual case).
+ * This also permits to gain some speedup by using firewall-based ACLs instead of
+ * rsyslog ACLs (the recommended method.
+ * rgerhards, 2009-11-16
+ */
+static rsRetVal
+HasRestrictions(uchar *pszType, int *bHasRestrictions) {
+ struct AllowedSenders *pAllowRoot;
+ DEFiRet;
+
+ CHKiRet(setAllowRoot(&pAllowRoot, pszType));
+
+ *bHasRestrictions = (pAllowRoot == NULL) ? 0 : 1;
+
+finalize_it:
+ if(iRet != RS_RET_OK) {
+ *bHasRestrictions = 1; /* in this case it is better to check individually */
+ DBGPRINTF("Error %d trying to obtain ACL restriction state of '%s'\n", iRet, pszType);
+ }
+ RETiRet;
+}
+
+
/* queryInterface function
* rgerhards, 2008-03-05
*/
@@ -1562,12 +1604,14 @@ CODESTARTobjQueryInterface(net)
pIf->create_udp_socket = create_udp_socket;
pIf->closeUDPListenSockets = closeUDPListenSockets;
pIf->isAllowedSender = isAllowedSender;
+ pIf->isAllowedSender2 = isAllowedSender2;
pIf->should_use_so_bsdcompat = should_use_so_bsdcompat;
pIf->getLocalHostname = getLocalHostname;
pIf->AddPermittedPeer = AddPermittedPeer;
pIf->DestructPermittedPeers = DestructPermittedPeers;
pIf->PermittedPeerWildcardMatch = PermittedPeerWildcardMatch;
pIf->CmpHost = CmpHost;
+ pIf->HasRestrictions = HasRestrictions;
/* data members */
pIf->pACLAddHostnameOnFail = &ACLAddHostnameOnFail;
pIf->pACLDontResolve = &ACLDontResolve;
diff --git a/runtime/net.h b/runtime/net.h
index ec364b1c..101ce79d 100644
--- a/runtime/net.h
+++ b/runtime/net.h
@@ -139,7 +139,7 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */
void (*debugListenInfo)(int fd, char *type);
int *(*create_udp_socket)(uchar *hostname, uchar *LogPort, int bIsServer);
void (*closeUDPListenSockets)(int *finet);
- int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost);
+ int (*isAllowedSender)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost); /* deprecated! */
rsRetVal (*getLocalHostname)(uchar**);
int (*should_use_so_bsdcompat)(void);
/* permitted peer handling should be replaced by something better (see comments above) */
@@ -148,11 +148,14 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */
rsRetVal (*PermittedPeerWildcardMatch)(permittedPeers_t *pPeer, uchar *pszNameToMatch, int *pbIsMatching);
/* v5 interface additions */
int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t);
+ /* v6 interface additions - 2009-11-16 */
+ rsRetVal (*HasRestrictions)(uchar *, int *bHasRestrictions);
+ int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS);
/* data members - these should go away over time... TODO */
int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */
int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */
ENDinterface(net)
-#define netCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */
+#define netCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */
/* prototypes */
PROTOTYPEObj(net);
diff --git a/runtime/netstrms.c b/runtime/netstrms.c
index 6b28e7ea..ea2dd9f3 100644
--- a/runtime/netstrms.c
+++ b/runtime/netstrms.c
@@ -36,9 +36,11 @@
#include "nsd.h"
#include "netstrm.h"
#include "nssel.h"
+#include "nspoll.h"
#include "netstrms.h"
MODULE_TYPE_LIB
+MODULE_TYPE_NOKEEP
/* static data */
DEFobjStaticHelpers
@@ -304,6 +306,7 @@ ENDObjClassInit(netstrms)
BEGINmodExit
CODESTARTmodExit
nsselClassExit();
+ nspollClassExit();
netstrmsClassExit();
netstrmClassExit(); /* we use this object, so we must exit it after we are finished */
ENDmodExit
@@ -322,6 +325,7 @@ CODESTARTmodInit
/* Initialize all classes that are in our module - this includes ourselfs */
CHKiRet(netstrmClassInit(pModInfo));
CHKiRet(nsselClassInit(pModInfo));
+ CHKiRet(nspollClassInit(pModInfo));
CHKiRet(netstrmsClassInit(pModInfo));
ENDmodInit
/* vi:set ai:
diff --git a/runtime/nsd.h b/runtime/nsd.h
index 8668c934..e5b9320b 100644
--- a/runtime/nsd.h
+++ b/runtime/nsd.h
@@ -87,4 +87,13 @@ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */
ENDinterface(nsdsel)
#define nsdselCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
+/* interface for the epoll call */
+BEGINinterface(nsdpoll) /* name must also be changed in ENDinterface macro! */
+ rsRetVal (*Construct)(nsdpoll_t **ppThis);
+ rsRetVal (*Destruct)(nsdpoll_t **ppThis);
+ rsRetVal (*Ctl)(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op);
+ rsRetVal (*Wait)(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr);
+ENDinterface(nsdpoll)
+#define nsdpollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
+
#endif /* #ifndef INCLUDED_NSD_H */
diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c
index ee78488a..0f0e0862 100644
--- a/runtime/nsd_gtls.c
+++ b/runtime/nsd_gtls.c
@@ -44,6 +44,7 @@
#include "stringbuf.h"
#include "errmsg.h"
#include "net.h"
+#include "datetime.h"
#include "nsd_ptcp.h"
#include "nsdsel_gtls.h"
#include "nsd_gtls.h"
@@ -55,12 +56,14 @@
GCRY_THREAD_OPTION_PTHREAD_IMPL;
MODULE_TYPE_LIB
+MODULE_TYPE_KEEP
/* static data */
DEFobjStaticHelpers
DEFobjCurrIf(errmsg)
DEFobjCurrIf(glbl)
DEFobjCurrIf(net)
+DEFobjCurrIf(datetime)
DEFobjCurrIf(nsd_ptcp)
static int bGlblSrvrInitDone = 0; /**< 0 - server global init not yet done, 1 - already done */
@@ -129,7 +132,7 @@ readFile(uchar *pszFile, gnutls_datum_t *pBuf)
ABORT_FINALIZE(RS_RET_FILE_TOO_LARGE);
}
- CHKmalloc(pBuf->data = malloc(stat_st.st_size));
+ CHKmalloc(pBuf->data = MALLOC(stat_st.st_size));
pBuf->size = stat_st.st_size;
if(read(fd, pBuf->data, stat_st.st_size) != stat_st.st_size) {
errmsg.LogError(0, RS_RET_IO_ERROR, "error or incomplete read of file '%s'", pszFile);
@@ -1020,7 +1023,7 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis)
}
/* get current time for certificate validation */
- if(time(&ttNow) == -1)
+ if(datetime.GetTime(&ttNow) == -1)
ABORT_FINALIZE(RS_RET_SYS_ERR);
/* as it looks, we need to validate the expiration dates ourselves...
@@ -1489,7 +1492,7 @@ Rcv(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf)
if(pThis->pszRcvBuf == NULL) {
/* we have no buffer, so we need to malloc one */
- CHKmalloc(pThis->pszRcvBuf = malloc(NSD_GTLS_MAX_RCVBUF));
+ CHKmalloc(pThis->pszRcvBuf = MALLOC(NSD_GTLS_MAX_RCVBUF));
pThis->lenRcvBuf = -1;
}
@@ -1704,6 +1707,7 @@ CODESTARTObjClassExit(nsd_gtls)
objRelease(nsd_ptcp, LM_NSD_PTCP_FILENAME);
objRelease(net, LM_NET_FILENAME);
objRelease(glbl, CORE_COMPONENT);
+ objRelease(datetime, CORE_COMPONENT);
objRelease(errmsg, CORE_COMPONENT);
ENDObjClassExit(nsd_gtls)
@@ -1715,6 +1719,7 @@ ENDObjClassExit(nsd_gtls)
BEGINObjClassInit(nsd_gtls, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */
/* request objects we use */
CHKiRet(objUse(errmsg, CORE_COMPONENT));
+ CHKiRet(objUse(datetime, CORE_COMPONENT));
CHKiRet(objUse(glbl, CORE_COMPONENT));
CHKiRet(objUse(net, LM_NET_FILENAME));
CHKiRet(objUse(nsd_ptcp, LM_NSD_PTCP_FILENAME));
diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c
index 54ee0666..69eb7684 100644
--- a/runtime/nsd_ptcp.c
+++ b/runtime/nsd_ptcp.c
@@ -48,9 +48,11 @@
#include "netstrms.h"
#include "netstrm.h"
#include "nsdsel_ptcp.h"
+#include "nsdpoll_ptcp.h"
#include "nsd_ptcp.h"
MODULE_TYPE_LIB
+MODULE_TYPE_NOKEEP
/* static data */
DEFobjStaticHelpers
@@ -296,12 +298,12 @@ FillRemHost(nsd_ptcp_t *pThis, struct sockaddr *pAddr)
* memory consumption)
*/
len = strlen((char*)szIP) + 1; /* +1 for \0 byte */
- if((pThis->pRemHostIP = malloc(len)) == NULL)
+ if((pThis->pRemHostIP = MALLOC(len)) == NULL)
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
memcpy(pThis->pRemHostIP, szIP, len);
len = strlen((char*)szHname) + 1; /* +1 for \0 byte */
- if((pThis->pRemHostName = malloc(len)) == NULL) {
+ if((pThis->pRemHostName = MALLOC(len)) == NULL) {
free(pThis->pRemHostIP); /* prevent leak */
pThis->pRemHostIP = NULL;
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
@@ -332,6 +334,12 @@ AcceptConnReq(nsd_t *pNsd, nsd_t **ppNew)
iNewSock = accept(pThis->sock, (struct sockaddr*) &addr, &addrlen);
if(iNewSock < 0) {
+ if(Debug) {
+ char errStr[1024];
+ rs_strerror_r(errno, errStr, sizeof(errStr));
+ dbgprintf("nds_ptcp: error accepting connection on socket %d, errno %d: %s\n",
+ pThis->sock, errno, errStr);
+ }
ABORT_FINALIZE(RS_RET_ACCEPT_ERR);
}
@@ -562,6 +570,7 @@ finalize_it:
static rsRetVal
Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf)
{
+ char errStr[1024];
DEFiRet;
nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd;
ISOBJ_TYPE_assert(pThis, nsd_ptcp);
@@ -571,7 +580,9 @@ Rcv(nsd_t *pNsd, uchar *pRcvBuf, ssize_t *pLenBuf)
if(*pLenBuf == 0) {
ABORT_FINALIZE(RS_RET_CLOSED);
} else if (*pLenBuf < 0) {
- ABORT_FINALIZE(RS_RET_ERR);
+ rs_strerror_r(errno, errStr, sizeof(errStr));
+ dbgprintf("error during recv on NSD %p: %s\n", pNsd, errStr);
+ ABORT_FINALIZE(RS_RET_RCV_ERR);
}
finalize_it:
@@ -821,6 +832,9 @@ ENDObjClassInit(nsd_ptcp)
BEGINmodExit
CODESTARTmodExit
+# ifdef HAVE_EPOLL_CREATE /* module only available if epoll() is supported! */
+ nsdpoll_ptcpClassExit();
+# endif
nsdsel_ptcpClassExit();
nsd_ptcpClassExit();
ENDmodExit
@@ -839,6 +853,9 @@ CODESTARTmodInit
/* Initialize all classes that are in our module - this includes ourselfs */
CHKiRet(nsd_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */
CHKiRet(nsdsel_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */
+# ifdef HAVE_EPOLL_CREATE /* module only available if epoll() is supported! */
+ CHKiRet(nsdpoll_ptcpClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */
+# endif
ENDmodInit
/* vi:set ai:
*/
diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c
new file mode 100644
index 00000000..ef9c37a3
--- /dev/null
+++ b/runtime/nsdpoll_ptcp.c
@@ -0,0 +1,290 @@
+/* nsdpoll_ptcp.c
+ *
+ * An implementation of the nsd epoll() interface for plain tcp sockets.
+ *
+ * Copyright 2009 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#include "config.h"
+
+#ifdef HAVE_EPOLL_CREATE /* this module requires epoll! */
+
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+#if HAVE_SYS_EPOLL_H
+# include <sys/epoll.h>
+#endif
+
+#include "rsyslog.h"
+#include "module-template.h"
+#include "obj.h"
+#include "errmsg.h"
+#include "srUtils.h"
+#include "nspoll.h"
+#include "nsd_ptcp.h"
+#include "nsdpoll_ptcp.h"
+
+/* static data */
+DEFobjStaticHelpers
+DEFobjCurrIf(errmsg)
+DEFobjCurrIf(glbl)
+
+
+/* -START------------------------- helpers for event list ------------------------------------ */
+
+/* add new entry to list. We assume that the fd is not already present and DO NOT check this!
+ * Returns newly created entry in pEvtLst.
+ * Note that we currently need to use level-triggered mode, because the upper layers do not work
+ * in parallel. As such, in edge-triggered mode we may not get notified, because new data comes
+ * in after we have read everything that was present. To use ET mode, we need to change the upper
+ * peers so that they immediately start a new wait before processing the data read. That obviously
+ * requires more elaborate redesign and we postpone this until the current more simplictic mode has
+ * been proven OK in practice.
+ * rgerhards, 2009-11-18
+ */
+static inline rsRetVal
+addEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, int mode, nsd_ptcp_t *pSock, nsdpoll_epollevt_lst_t **pEvtLst) {
+ nsdpoll_epollevt_lst_t *pNew;
+ DEFiRet;
+
+ CHKmalloc(pNew = (nsdpoll_epollevt_lst_t*) malloc(sizeof(nsdpoll_epollevt_lst_t)));
+ pNew->id = id;
+ pNew->pUsr = pUsr;
+ pNew->pSock = pSock;
+ pNew->event.events = 0; /* TODO: at some time we should be able to use EPOLLET */
+ if(mode & NSDPOLL_IN)
+ pNew->event.events |= EPOLLIN;
+ if(mode & NSDPOLL_OUT)
+ pNew->event.events |= EPOLLOUT;
+ pNew->event.data.u64 = (uint64) pNew;
+ pNew->pNext = pThis->pRoot;
+ pThis->pRoot = pNew;
+ *pEvtLst = pNew;
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* find and unlink the entry identified by id/pUsr from the list.
+ * rgerhards, 2009-11-23
+ */
+static inline rsRetVal
+unlinkEvent(nsdpoll_ptcp_t *pThis, int id, void *pUsr, nsdpoll_epollevt_lst_t **ppEvtLst) {
+ nsdpoll_epollevt_lst_t *pEvtLst;
+ nsdpoll_epollevt_lst_t *pPrev = NULL;
+ DEFiRet;
+
+ pEvtLst = pThis->pRoot;
+ while(pEvtLst != NULL && !(pEvtLst->id == id && pEvtLst->pUsr == pUsr)) {
+ pPrev = pEvtLst;
+ pEvtLst = pEvtLst->pNext;
+ }
+ if(pEvtLst == NULL)
+ ABORT_FINALIZE(RS_RET_NOT_FOUND);
+
+ *ppEvtLst = pEvtLst;
+
+ /* unlink */
+ if(pPrev == NULL)
+ pThis->pRoot = pEvtLst->pNext;
+ else
+ pPrev->pNext = pEvtLst->pNext;
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* destruct the provided element. It must already be unlinked from the list.
+ * rgerhards, 2009-11-23
+ */
+static inline rsRetVal
+delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) {
+ DEFiRet;
+ free(*ppEvtLst);
+ *ppEvtLst = NULL;
+ RETiRet;
+}
+
+
+/* -END--------------------------- helpers for event list ------------------------------------ */
+
+
+/* Standard-Constructor
+ */
+BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */
+#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1)
+ DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n");
+ pThis->efd = epoll_create1(EPOLL_CLOEXEC);
+ if(pThis->efd < 0 && errno == ENOSYS)
+#endif
+ {
+ DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n");
+ pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */
+ }
+
+ if(pThis->efd < 0) {
+ DBGPRINTF("epoll_create1() could not create fd\n");
+ ABORT_FINALIZE(RS_RET_IO_ERROR);
+ }
+finalize_it:
+ENDobjConstruct(nsdpoll_ptcp)
+
+
+/* destructor for the nsdpoll_ptcp object */
+BEGINobjDestruct(nsdpoll_ptcp) /* be sure to specify the object type also in END and CODESTART macros! */
+CODESTARTobjDestruct(nsdpoll_ptcp)
+ENDobjDestruct(nsdpoll_ptcp)
+
+
+/* Modify socket set */
+static rsRetVal
+Ctl(nsdpoll_t *pNsdpoll, nsd_t *pNsd, int id, void *pUsr, int mode, int op) {
+ nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll;
+ nsd_ptcp_t *pSock = (nsd_ptcp_t*) pNsd;
+ nsdpoll_epollevt_lst_t *pEventLst;
+ int errSave;
+ char errStr[512];
+ DEFiRet;
+
+ if(op == NSDPOLL_ADD) {
+ dbgprintf("adding nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock);
+ CHKiRet(addEvent(pThis, id, pUsr, mode, pSock, &pEventLst));
+ if(epoll_ctl(pThis->efd, EPOLL_CTL_ADD, pSock->sock, &pEventLst->event) < 0) {
+ errSave = errno;
+ rs_strerror_r(errSave, errStr, sizeof(errStr));
+ errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL,
+ "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n",
+ pSock->sock, id, pUsr, mode, errStr);
+ }
+ } else if(op == NSDPOLL_DEL) {
+ dbgprintf("removing nsdpoll entry %d/%p, sock %d\n", id, pUsr, pSock->sock);
+ CHKiRet(unlinkEvent(pThis, id, pUsr, &pEventLst));
+ if(epoll_ctl(pThis->efd, EPOLL_CTL_DEL, pSock->sock, &pEventLst->event) < 0) {
+ errSave = errno;
+ rs_strerror_r(errSave, errStr, sizeof(errStr));
+ errmsg.LogError(errSave, RS_RET_ERR_EPOLL_CTL,
+ "epoll_ctl failed on fd %d, id %d/%p, op %d with %s\n",
+ pSock->sock, id, pUsr, mode, errStr);
+ ABORT_FINALIZE(RS_RET_ERR_EPOLL_CTL);
+ }
+ CHKiRet(delEvent(&pEventLst));
+ } else {
+ dbgprintf("program error: invalid NSDPOLL_mode %d - ignoring request\n", op);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* Wait for io to become ready. After the successful call, idRdy contains the
+ * id set by the caller for that i/o event, ppUsr is a pointer to a location
+ * where the user pointer shall be stored.
+ * TODO: this is a trivial implementation that only polls one event at a time. We
+ * may later extend it to poll for multiple events, what would cause less
+ * overhead.
+ * rgerhards, 2009-11-18
+ */
+static rsRetVal
+Wait(nsdpoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr) {
+ nsdpoll_ptcp_t *pThis = (nsdpoll_ptcp_t*) pNsdpoll;
+ nsdpoll_epollevt_lst_t *pOurEvt;
+ struct epoll_event event;
+ int nfds;
+ DEFiRet;
+
+ assert(idRdy != NULL);
+ assert(ppUsr != NULL);
+
+ nfds = epoll_wait(pThis->efd, &event, 1, timeout);
+ if(nfds == -1) {
+ if(errno == EINTR) {
+ ABORT_FINALIZE(RS_RET_EINTR);
+ } else {
+ DBGPRINTF("epoll() returned with error code %d\n", errno);
+ ABORT_FINALIZE(RS_RET_ERR_EPOLL);
+ }
+ } else if(nfds == 0) {
+ ABORT_FINALIZE(RS_RET_TIMEOUT);
+ }
+
+ /* we got a valid event, so tell the caller... */
+ pOurEvt = (nsdpoll_epollevt_lst_t*) event.data.u64;
+ *idRdy = pOurEvt->id;
+ *ppUsr = pOurEvt->pUsr;
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* ------------------------------ end support for the epoll() interface ------------------------------ */
+
+
+/* queryInterface function */
+BEGINobjQueryInterface(nsdpoll_ptcp)
+CODESTARTobjQueryInterface(nsdpoll_ptcp)
+ if(pIf->ifVersion != nsdCURR_IF_VERSION) {/* check for current version, increment on each change */
+ ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED);
+ }
+
+ /* ok, we have the right interface, so let's fill it
+ * Please note that we may also do some backwards-compatibility
+ * work here (if we can support an older interface version - that,
+ * of course, also affects the "if" above).
+ */
+ pIf->Construct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpConstruct;
+ pIf->Destruct = (rsRetVal(*)(nsdpoll_t**)) nsdpoll_ptcpDestruct;
+ pIf->Ctl = Ctl;
+ pIf->Wait = Wait;
+finalize_it:
+ENDobjQueryInterface(nsdpoll_ptcp)
+
+
+/* exit our class
+ */
+BEGINObjClassExit(nsdpoll_ptcp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */
+CODESTARTObjClassExit(nsdpoll_ptcp)
+ /* release objects we no longer need */
+ objRelease(glbl, CORE_COMPONENT);
+ objRelease(errmsg, CORE_COMPONENT);
+ENDObjClassExit(nsdpoll_ptcp)
+
+
+/* Initialize the nsdpoll_ptcp class. Must be called as the very first method
+ * before anything else is called inside this class.
+ * rgerhards, 2008-02-19
+ */
+BEGINObjClassInit(nsdpoll_ptcp, 1, OBJ_IS_CORE_MODULE) /* class, version */
+ /* request objects we use */
+ CHKiRet(objUse(errmsg, CORE_COMPONENT));
+ CHKiRet(objUse(glbl, CORE_COMPONENT));
+
+ /* set our own handlers */
+ENDObjClassInit(nsdpoll_ptcp)
+#endif /* #ifdef HAVE_EPOLL_CREATE this module requires epoll! */
+
+/* vi:set ai:
+ */
diff --git a/runtime/nsdpoll_ptcp.h b/runtime/nsdpoll_ptcp.h
new file mode 100644
index 00000000..cea2823d
--- /dev/null
+++ b/runtime/nsdpoll_ptcp.h
@@ -0,0 +1,60 @@
+/* An implementation of the nsd poll interface for plain tcp sockets.
+ *
+ * Copyright 2009 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+
+#ifndef INCLUDED_NSDPOLL_PTCP_H
+#define INCLUDED_NSDPOLL_PTCP_H
+
+#include "nsd.h"
+#if HAVE_SYS_EPOLL_H
+# include <sys/epoll.h>
+#endif
+typedef nsdpoll_if_t nsdpoll_ptcp_if_t; /* we just *implement* this interface */
+/* a helper object to keep track of the epoll event records
+ * Note that we need to keep track of that list because we need to
+ * free the events when they are no longer needed.
+ */
+typedef struct nsdpoll_epollevt_lst_s nsdpoll_epollevt_lst_t;
+struct nsdpoll_epollevt_lst_s {
+#if HAVE_SYS_EPOLL_H
+ epoll_event_t event;
+#endif
+ int id;
+ void *pUsr;
+ nsd_ptcp_t *pSock; /* our associated netstream driver data */
+ nsdpoll_epollevt_lst_t *pNext;
+};
+
+/* the nsdpoll_ptcp object */
+struct nsdpoll_ptcp_s {
+ BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */
+ int efd; /* file descriptor used by epoll */
+ nsdpoll_epollevt_lst_t *pRoot; /* Root of the epoll event list */
+};
+
+/* interface is defined in nsd.h, we just implement it! */
+#define nsdpoll_ptcpCURR_IF_VERSION nsdCURR_IF_VERSION
+
+/* prototypes */
+PROTOTYPEObj(nsdpoll_ptcp);
+
+#endif /* #ifndef INCLUDED_NSDPOLL_PTCP_H */
diff --git a/runtime/nsdsel_gtls.c b/runtime/nsdsel_gtls.c
index 1a389a00..aff55af2 100644
--- a/runtime/nsdsel_gtls.c
+++ b/runtime/nsdsel_gtls.c
@@ -177,6 +177,7 @@ doRetry(nsd_gtls_t *pNsd)
finalize_it:
if(iRet != RS_RET_OK && iRet != RS_RET_CLOSED && iRet != RS_RET_RETRY)
pNsd->bAbortConn = 1; /* request abort */
+dbgprintf("XXXXXX: doRetry: iRet %d, pNsd->bAbortConn %d\n", iRet, pNsd->bAbortConn);
RETiRet;
}
diff --git a/runtime/nspoll.c b/runtime/nspoll.c
new file mode 100644
index 00000000..64927280
--- /dev/null
+++ b/runtime/nspoll.c
@@ -0,0 +1,195 @@
+/* nspoll.c
+ *
+ * This is an io waiter interface utilizing the much-more-efficient poll/epoll API.
+ * Note that it may not always be available for a given driver. If so, that is reported
+ * back to the upper peer which then should consult a nssel-based io waiter.
+ *
+ * Work on this module begun 2009-11-18 by Rainer Gerhards.
+ *
+ * Copyright 2009 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#include "config.h"
+
+#include "rsyslog.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+
+#include "rsyslog.h"
+#include "obj.h"
+#include "module-template.h"
+#include "netstrm.h"
+#include "nspoll.h"
+
+/* static data */
+DEFobjStaticHelpers
+DEFobjCurrIf(glbl)
+
+
+/* load our low-level driver. This must be done before any
+ * driver-specific functions (allmost all...) can be carried
+ * out. Note that the driver's .ifIsLoaded is correctly
+ * initialized by calloc() and we depend on that. Please note that
+ * we do some name-mangeling. We know that each nsd driver also needs
+ * a nspoll driver. So we simply append "sel" to the nsd driver name: This,
+ * of course, means that the driver name must match these rules, but that
+ * shouldn't be a real problem.
+ * WARNING: this code is mostly identical to similar code in
+ * netstrms.c - TODO: abstract it and move it to some common place.
+ * rgerhards, 2008-04-28
+ */
+static rsRetVal
+loadDrvr(nspoll_t *pThis)
+{
+ DEFiRet;
+ uchar *pBaseDrvrName;
+ uchar szDrvrName[48]; /* 48 shall be large enough */
+
+ pBaseDrvrName = pThis->pBaseDrvrName;
+ if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */
+ pBaseDrvrName = glbl.GetDfltNetstrmDrvr();
+ if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdpoll_%s", pBaseDrvrName) == sizeof(szDrvrName))
+ ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG);
+ CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName));
+
+ pThis->Drvr.ifVersion = nsdCURR_IF_VERSION;
+ /* The pDrvrName+2 below is a hack to obtain the object name. It
+ * safes us to have yet another variable with the name without "lm" in
+ * front of it. If we change the module load interface, we may re-think
+ * about this hack, but for the time being it is efficient and clean
+ * enough. -- rgerhards, 2008-04-18
+ */
+ CHKiRet(obj.UseObj(__FILE__, szDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr));
+
+finalize_it:
+ if(iRet != RS_RET_OK) {
+ if(pThis->pDrvrName != NULL)
+ free(pThis->pDrvrName);
+ pThis->pDrvrName = NULL;
+ }
+ RETiRet;
+}
+
+
+/* Standard-Constructor */
+BEGINobjConstruct(nspoll) /* be sure to specify the object type also in END macro! */
+ENDobjConstruct(nspoll)
+
+
+/* destructor for the nspoll object */
+BEGINobjDestruct(nspoll) /* be sure to specify the object type also in END and CODESTART macros! */
+CODESTARTobjDestruct(nspoll)
+ if(pThis->pDrvrData != NULL)
+ pThis->Drvr.Destruct(&pThis->pDrvrData);
+
+ /* and now we must release our driver, if we got one. We use the presence of
+ * a driver name string as load indicator (because we also need that string
+ * to release the driver
+ */
+ if(pThis->pDrvrName != NULL) {
+ obj.ReleaseObj(__FILE__, pThis->pDrvrName+2, DONT_LOAD_LIB, (void*) &pThis->Drvr);
+ free(pThis->pDrvrName);
+ }
+ENDobjDestruct(nspoll)
+
+
+/* ConstructionFinalizer */
+static rsRetVal
+ConstructFinalize(nspoll_t *pThis)
+{
+ DEFiRet;
+ ISOBJ_TYPE_assert(pThis, nspoll);
+ CHKiRet(loadDrvr(pThis));
+ CHKiRet(pThis->Drvr.Construct(&pThis->pDrvrData));
+finalize_it:
+ RETiRet;
+}
+
+
+/* Carries out the actual wait (all done in lower layers)
+ */
+static rsRetVal
+Wait(nspoll_t *pThis, int timeout, int *idRdy, void **ppUsr) {
+ DEFiRet;
+ ISOBJ_TYPE_assert(pThis, nspoll);
+ assert(idRdy != NULL);
+ iRet = pThis->Drvr.Wait(pThis->pDrvrData, timeout, idRdy, ppUsr);
+ RETiRet;
+}
+
+
+/* semantics like the epoll_ctl() function, does the same thing.
+ * rgerhards, 2009-11-18
+ */
+static rsRetVal
+Ctl(nspoll_t *pThis, netstrm_t *pStrm, int id, void *pUsr, int mode, int op) {
+ DEFiRet;
+ ISOBJ_TYPE_assert(pThis, nspoll);
+ iRet = pThis->Drvr.Ctl(pThis->pDrvrData, pStrm->pDrvrData, id, pUsr, mode, op);
+ RETiRet;
+}
+
+
+/* queryInterface function */
+BEGINobjQueryInterface(nspoll)
+CODESTARTobjQueryInterface(nspoll)
+ if(pIf->ifVersion != nspollCURR_IF_VERSION) {/* check for current version, increment on each change */
+ ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED);
+ }
+
+ /* ok, we have the right interface, so let's fill it
+ * Please note that we may also do some backwards-compatibility
+ * work here (if we can support an older interface version - that,
+ * of course, also affects the "if" above).
+ */
+ pIf->Construct = nspollConstruct;
+ pIf->ConstructFinalize = ConstructFinalize;
+ pIf->Destruct = nspollDestruct;
+ pIf->Wait = Wait;
+ pIf->Ctl = Ctl;
+finalize_it:
+ENDobjQueryInterface(nspoll)
+
+
+/* exit our class
+ */
+BEGINObjClassExit(nspoll, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */
+CODESTARTObjClassExit(nspoll)
+ /* release objects we no longer need */
+ objRelease(glbl, CORE_COMPONENT);
+ENDObjClassExit(nspoll)
+
+
+/* Initialize the nspoll class. Must be called as the very first method
+ * before anything else is called inside this class.
+ * rgerhards, 2008-02-19
+ */
+BEGINObjClassInit(nspoll, 1, OBJ_IS_CORE_MODULE) /* class, version */
+ /* request objects we use */
+ DBGPRINTF("doing nspollClassInit\n");
+ CHKiRet(objUse(glbl, CORE_COMPONENT));
+
+ /* set our own handlers */
+ENDObjClassInit(nspoll)
+/* vi:set ai:
+ */
diff --git a/runtime/nspoll.h b/runtime/nspoll.h
new file mode 100644
index 00000000..a77759c0
--- /dev/null
+++ b/runtime/nspoll.h
@@ -0,0 +1,65 @@
+/* Definitions for the nspoll io activity waiter
+ *
+ * Copyright 2009 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+
+#ifndef INCLUDED_NSPOLL_H
+#define INCLUDED_NSPOLL_H
+
+#include "netstrms.h"
+
+/* some operations to be portable when we do not have epoll() available */
+#define NSDPOLL_ADD 1
+#define NSDPOLL_DEL 2
+
+/* and some mode specifiers for waiting on input/output */
+#define NSDPOLL_IN 1 /* EPOLLIN */
+#define NSDPOLL_OUT 2 /* EPOLLOUT */
+/* next is 4, 8, 16, ... - must be bit values, as they are ored! */
+
+/* the nspoll object */
+struct nspoll_s {
+ BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */
+ nsd_t *pDrvrData; /**< the driver's data elements */
+ uchar *pBaseDrvrName; /**< nsd base driver name to use, or NULL if system default */
+ uchar *pDrvrName; /**< full base driver name (set when driver is loaded) */
+ nsdpoll_if_t Drvr; /**< our stream driver */
+};
+
+
+/* interface */
+BEGINinterface(nspoll) /* name must also be changed in ENDinterface macro! */
+ rsRetVal (*Construct)(nspoll_t **ppThis);
+ rsRetVal (*ConstructFinalize)(nspoll_t *pThis);
+ rsRetVal (*Destruct)(nspoll_t **ppThis);
+ rsRetVal (*Wait)(nspoll_t *pNsdpoll, int timeout, int *idRdy, void **ppUsr);
+ rsRetVal (*Ctl)(nspoll_t *pNsdpoll, netstrm_t *pStrm, int id, void *pUsr, int mode, int op);
+ rsRetVal (*IsEPollSupported)(void); /* static method */
+ENDinterface(nspoll)
+#define nspollCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
+
+/* prototypes */
+PROTOTYPEObj(nspoll);
+
+/* the name of our library binary */
+#define LM_NSPOLL_FILENAME LM_NETSTRMS_FILENAME
+
+#endif /* #ifndef INCLUDED_NSPOLL_H */
diff --git a/runtime/nssel.c b/runtime/nssel.c
index d11d5fe1..7c5be3a9 100644
--- a/runtime/nssel.c
+++ b/runtime/nssel.c
@@ -219,6 +219,7 @@ ENDObjClassExit(nssel)
*/
BEGINObjClassInit(nssel, 1, OBJ_IS_CORE_MODULE) /* class, version */
/* request objects we use */
+ DBGPRINTF("doing nsselClassInit\n");
CHKiRet(objUse(glbl, CORE_COMPONENT));
/* set our own handlers */
diff --git a/runtime/obj.c b/runtime/obj.c
index aebea332..45dac776 100644
--- a/runtime/obj.c
+++ b/runtime/obj.c
@@ -48,7 +48,7 @@
*
* File begun on 2008-01-04 by RGerhards
*
- * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -90,6 +90,7 @@
#include "cfsysline.h"
#include "unicode-helper.h"
#include "apc.h"
+#include "datetime.h"
/* static data */
DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */
@@ -1129,7 +1130,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf)
/* DEV debug only: dbgprintf("source file %s requests object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */
- d_pthread_mutex_lock(&mutObjGlobalOp);
+ pthread_mutex_lock(&mutObjGlobalOp);
if(pIf->ifIsLoaded == 1) {
ABORT_FINALIZE(RS_RET_OK); /* we are already set */
@@ -1170,7 +1171,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf)
pIf->ifIsLoaded = 1; /* we are happy */
finalize_it:
- d_pthread_mutex_unlock(&mutObjGlobalOp);
+ pthread_mutex_unlock(&mutObjGlobalOp);
if(pStr != NULL)
rsCStrDestruct(&pStr);
@@ -1193,7 +1194,7 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf)
/* dev debug only dbgprintf("source file %s releasing object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */
- d_pthread_mutex_lock(&mutObjGlobalOp);
+ pthread_mutex_lock(&mutObjGlobalOp);
if(pObjFile == NULL)
FINALIZE; /* if it is not a lodable module, we do not need to do anything... */
@@ -1214,7 +1215,7 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf)
pIf->ifIsLoaded = 0; /* indicated "no longer valid" */
finalize_it:
- d_pthread_mutex_unlock(&mutObjGlobalOp);
+ pthread_mutex_unlock(&mutObjGlobalOp);
if(pStr != NULL)
rsCStrDestruct(&pStr);
@@ -1330,8 +1331,9 @@ objClassInit(modInfo_t *pModInfo)
CHKiRet(objGetObjInterface(&obj)); /* get ourselves ;) */
/* init classes we use (limit to as few as possible!) */
- CHKiRet(apcClassInit(pModInfo));
CHKiRet(errmsgClassInit(pModInfo));
+ CHKiRet(datetimeClassInit(pModInfo));
+ CHKiRet(apcClassInit(pModInfo));
CHKiRet(cfsyslineInit());
CHKiRet(varClassInit(pModInfo));
CHKiRet(moduleClassInit(pModInfo));
diff --git a/runtime/objomsr.c b/runtime/objomsr.c
index 8dddc4b8..1d442c61 100644
--- a/runtime/objomsr.c
+++ b/runtime/objomsr.c
@@ -67,31 +67,25 @@ rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries)
assert(ppThis != NULL);
assert(iNumEntries >= 0);
- if((pThis = calloc(1, sizeof(omodStringRequest_t))) == NULL) {
- iRet = RS_RET_OUT_OF_MEMORY;
- goto abort_it;
- }
+ CHKmalloc(pThis = calloc(1, sizeof(omodStringRequest_t)));
/* got the structure, so fill it */
pThis->iNumEntries = iNumEntries;
/* allocate string for template name array. The individual strings will be
* allocated as the code progresses (we do not yet know the string sizes)
*/
- if((pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))) == NULL) {
- OMSRdestruct(pThis);
- pThis = NULL;
- iRet = RS_RET_OUT_OF_MEMORY;
- goto abort_it;
- }
+ CHKmalloc(pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*)));
+
/* allocate the template options array. */
- if((pThis->piTplOpts = calloc(iNumEntries, sizeof(int))) == NULL) {
- OMSRdestruct(pThis);
- pThis = NULL;
- iRet = RS_RET_OUT_OF_MEMORY;
- goto abort_it;
- }
+ CHKmalloc(pThis->piTplOpts = calloc(iNumEntries, sizeof(int)));
-abort_it:
+finalize_it:
+ if(iRet != RS_RET_OK) {
+ if(pThis != NULL) {
+ OMSRdestruct(pThis);
+ pThis = NULL;
+ }
+ }
*ppThis = pThis;
RETiRet;
}
@@ -155,7 +149,7 @@ OMSRgetSupportedTplOpts(unsigned long *pOpts)
{
DEFiRet;
assert(pOpts != NULL);
- *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY;
+ *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY | OMSR_TPL_AS_MSG;
RETiRet;
}
diff --git a/runtime/objomsr.h b/runtime/objomsr.h
index 75ad0fb8..e59b774f 100644
--- a/runtime/objomsr.h
+++ b/runtime/objomsr.h
@@ -27,8 +27,12 @@
/* define flags for required template options */
#define OMSR_NO_RQD_TPL_OPTS 0
#define OMSR_RQD_TPL_OPT_SQL 1
+/* only one of OMSR_TPL_AS_ARRAY or _AS_MSG must be specified, if both are given
+ * results are unpredictable.
+ */
#define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */
-/* next option is 4, 8, 16, ... */
+#define OMSR_TPL_AS_MSG 4 /* introduced in 5.3.4, 2009-11-02 */
+/* next option is 8, 16, 32, ... */
struct omodStringRequest_s { /* strings requested by output module for doAction() */
int iNumEntries; /* number of array entries for data elements below */
diff --git a/runtime/parser.c b/runtime/parser.c
index be9304d7..b385c54b 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -37,7 +37,13 @@
#include "dirty.h"
#include "msg.h"
#include "obj.h"
+#include "datetime.h"
#include "errmsg.h"
+#include "parser.h"
+#include "ruleset.h"
+#include "unicode-helper.h"
+#include "dirty.h"
+#include "cfsysline.h"
/* some defines */
#define DEFUPRI (LOG_USER|LOG_NOTICE)
@@ -46,25 +52,166 @@
DEFobjStaticHelpers
DEFobjCurrIf(glbl)
DEFobjCurrIf(errmsg)
+DEFobjCurrIf(datetime)
+DEFobjCurrIf(ruleset)
/* static data */
+/* config data */
+static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */
+static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */
+static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */
+static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */
+static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */
+
+/* This is the list of all parsers known to us.
+ * This is also used to unload all modules on shutdown.
+ */
+parserList_t *pParsLstRoot = NULL;
+
+/* this is the list of the default parsers, to be used if no others
+ * are specified.
+ */
+parserList_t *pDfltParsLst = NULL;
+
+
+/* intialize (but NOT allocate) a parser list. Primarily meant as a hook
+ * which can be used to extend the list in the future. So far, just sets
+ * it to NULL.
+ */
+static rsRetVal
+InitParserList(parserList_t **pListRoot)
+{
+ *pListRoot = NULL;
+ return RS_RET_OK;
+}
+
+
+/* destruct a parser list. The list elements are destroyed, but the parser objects
+ * themselves are not modified. (That is done at a late stage during rsyslogd
+ * shutdown and need not be considered here.)
+ */
+static rsRetVal
+DestructParserList(parserList_t **ppListRoot)
+{
+ parserList_t *pParsLst;
+ parserList_t *pParsLstDel;
+
+ pParsLst = *ppListRoot;
+ while(pParsLst != NULL) {
+ pParsLstDel = pParsLst;
+ pParsLst = pParsLst->pNext;
+ free(pParsLstDel);
+ }
+ *ppListRoot = NULL;
+ return RS_RET_OK;
+}
+
-/* this is a dummy class init
+/* Add a parser to the list. We use a VERY simple and ineffcient algorithm,
+ * but it is employed only for a few milliseconds during config processing. So
+ * I prefer to keep it very simple and with simple data structures. Unfortunately,
+ * we need to preserve the order, but I don't like to add a tail pointer as that
+ * would require a container object. So I do the extra work to skip to the tail
+ * when adding elements...
+ * rgerhards, 2009-11-03
*/
-rsRetVal parserClassInit(void)
+static rsRetVal
+AddParserToList(parserList_t **ppListRoot, parser_t *pParser)
{
+ parserList_t *pThis;
+ parserList_t *pTail;
DEFiRet;
- /* request objects we use */
- CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */
- CHKiRet(objUse(glbl, CORE_COMPONENT));
- CHKiRet(objUse(errmsg, CORE_COMPONENT));
-// TODO: free components! see action.c
+ CHKmalloc(pThis = MALLOC(sizeof(parserList_t)));
+ pThis->pParser = pParser;
+ pThis->pNext = NULL;
+
+ if(*ppListRoot == NULL) {
+ pThis->pNext = *ppListRoot;
+ *ppListRoot = pThis;
+ } else {
+ /* find tail first */
+ for(pTail = *ppListRoot ; pTail->pNext != NULL ; pTail = pTail->pNext)
+ /* just search, do nothing else */;
+ /* add at tail */
+ pTail->pNext = pThis;
+ }
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* find a parser based on the provided name */
+static rsRetVal
+FindParser(parser_t **ppParser, uchar *pName)
+{
+ parserList_t *pThis;
+ DEFiRet;
+
+ for(pThis = pParsLstRoot ; pThis != NULL ; pThis = pThis->pNext) {
+ if(ustrcmp(pThis->pParser->pName, pName) == 0) {
+ *ppParser = pThis->pParser;
+ FINALIZE; /* found it, iRet still eq. OK! */
+ }
+ }
+
+ iRet = RS_RET_PARSER_NOT_FOUND;
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* --- END helper functions for parser list handling --- */
+
+/* Add a an already existing parser to the default list. As usual, order
+ * of calls is important (most importantly, that means the legacy parser,
+ * which can process everything, MUST be added last!).
+ * rgerhards, 2009-11-04
+ */
+static rsRetVal
+AddDfltParser(uchar *pName)
+{
+ parser_t *pParser;
+ DEFiRet;
+
+ CHKiRet(FindParser(&pParser, pName));
+ CHKiRet(AddParserToList(&pDfltParsLst, pParser));
+ dbgprintf("Parser '%s' added to default parser set.\n", pName);
+
+finalize_it:
+ RETiRet;
+}
+
+
+
+BEGINobjConstruct(parser) /* be sure to specify the object type also in END macro! */
+ENDobjConstruct(parser)
+
+/* ConstructionFinalizer. The most important chore is to add the parser object
+ * to our global list of available parsers.
+ * rgerhards, 2009-11-03
+ */
+rsRetVal parserConstructFinalize(parser_t *pThis)
+{
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, parser);
+ CHKiRet(AddParserToList(&pParsLstRoot, pThis));
+ DBGPRINTF("Parser '%s' added to list of available parsers.\n", pThis->pName);
+
finalize_it:
RETiRet;
}
+BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */
+CODESTARTobjDestruct(parser)
+ dbgprintf("destructing parser '%s'\n", pThis->pName);
+ free(pThis->pName);
+ENDobjDestruct(parser)
+
/* uncompress a received message if it is compressed.
* pMsg->pszRawMsg buffer is updated.
@@ -96,7 +243,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg)
*/
int ret;
iLenDefBuf = glbl.GetMaxLine();
- CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1)));
+ CHKmalloc(deflateBuf = MALLOC(sizeof(uchar) * (iLenDefBuf + 1)));
ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1);
DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n",
ret, (long) iLenDefBuf, (int) (lenMsg-1));
@@ -153,7 +300,7 @@ finalize_it:
* rgerhards, 2007-09-14
*/
static inline rsRetVal
-sanitizeMessage(msg_t *pMsg)
+SanitizeMsg(msg_t *pMsg)
{
DEFiRet;
uchar *pszMsg;
@@ -163,16 +310,12 @@ sanitizeMessage(msg_t *pMsg)
size_t iDst;
size_t iMaxLine;
size_t maxDest;
- bool bUpdatedLen = FALSE;
+ sbool bUpdatedLen = FALSE;
uchar szSanBuf[32*1024]; /* buffer used for sanitizing a string */
assert(pMsg != NULL);
assert(pMsg->iLenRawMsg > 0);
-# ifdef USE_NETZIP
- CHKiRet(uncompressMessage(pMsg));
-# endif
-
pszMsg = pMsg->pszRawMsg;
lenMsg = pMsg->iLenRawMsg;
@@ -202,6 +345,11 @@ sanitizeMessage(msg_t *pMsg)
* needs sanitation than to do the sanitation in any case. So we first do
* this and terminate when it is not needed - which is expectedly the case
* for the vast majority of messages. -- rgerhards, 2009-06-15
+ * Note that we do NOT check here if tab characters are to be escaped or
+ * not. I expect this functionality to be seldomly used and thus I do not
+ * like to pay the performance penalty. So the penalty is only with those
+ * that actually use it, because we may call the sanitizer without actual
+ * need below (but it then still will work perfectly well!). -- rgerhards, 2009-11-27
*/
int bNeedSanitize = 0;
for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) {
@@ -210,6 +358,9 @@ sanitizeMessage(msg_t *pMsg)
bNeedSanitize = 1;
break;
}
+ } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) {
+ bNeedSanitize = 1;
+ break;
}
}
@@ -227,23 +378,31 @@ sanitizeMessage(msg_t *pMsg)
if(maxDest < sizeof(szSanBuf))
pDst = szSanBuf;
else
- CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1)));
+ CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1)));
iSrc = iDst = 0;
while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */
- if(iscntrl((int) pszMsg[iSrc])) {
+ if(iscntrl((int) pszMsg[iSrc]) && (pszMsg[iSrc] != '\t' || bEscapeTab)) {
/* note: \0 must always be escaped, the rest of the code currently
* can not handle it! -- rgerhards, 2009-08-26
*/
if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) {
- /* we are configured to escape control characters. Please note
- * that this most probably break non-western character sets like
- * Japanese, Korean or Chinese. rgerhards, 2007-07-17
+ /* we are configured to escape control characters. Please note
+ * that this most probably break non-western character sets like
+ * Japanese, Korean or Chinese. rgerhards, 2007-07-17
+ */
+ pDst[iDst++] = cCCEscapeChar;
+ pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6);
+ pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3);
+ pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007));
+ }
+ } else if(pszMsg[iSrc] > 127 && bEscape8BitChars) {
+ /* In this case, we also do the conversion. Note that this most
+ * probably breaks European languages. -- rgerhards, 2010-01-27
*/
pDst[iDst++] = cCCEscapeChar;
pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6);
pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3);
pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007));
- }
} else {
pDst[iDst++] = pszMsg[iSrc];
}
@@ -260,26 +419,17 @@ finalize_it:
RETiRet;
}
-
-/* Parse a received message. The object's rawmsg property is taken and
- * parsed according to the relevant standards. This can later be
- * extended to support configured parsers.
- * rgerhards, 2008-10-09
+/* A standard parser to parse out the PRI. This is made available in
+ * this module as it is expected that allmost all parsers will need
+ * that functionality and so they do not need to implement it themsleves.
*/
-rsRetVal parseMsg(msg_t *pMsg)
+static inline rsRetVal
+ParsePRI(msg_t *pMsg)
{
- DEFiRet;
- uchar *msg;
int pri;
+ uchar *msg;
int lenMsg;
-
- if(pMsg->iLenRawMsg == 0)
- ABORT_FINALIZE(RS_RET_EMPTY_MSG);
-
- CHKiRet(sanitizeMessage(pMsg));
-
- /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */
- DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, getRcvFrom(pMsg), pMsg->pszRawMsg);
+ DEFiRet;
/* pull PRI */
lenMsg = pMsg->iLenRawMsg;
@@ -307,31 +457,252 @@ rsRetVal parseMsg(msg_t *pMsg)
pMsg->iSeverity = LOG_PRI(pri);
MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg);
}
+ RETiRet;
+}
+
+
+/* Parse a received message. The object's rawmsg property is taken and
+ * parsed according to the relevant standards. This can later be
+ * extended to support configured parsers.
+ * rgerhards, 2008-10-09
+ */
+static rsRetVal
+ParseMsg(msg_t *pMsg)
+{
+ rsRetVal localRet = RS_RET_ERR;
+ parserList_t *pParserList;
+ parser_t *pParser;
+ sbool bIsSanitized;
+ sbool bPRIisParsed;
+ static int iErrMsgRateLimiter = 0;
+ DEFiRet;
+
+ if(pMsg->iLenRawMsg == 0)
+ ABORT_FINALIZE(RS_RET_EMPTY_MSG);
- /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have
- * a traditional syslog message or one formatted according to syslog-protocol.
- * We need to apply different parsers depending on that. We use the
- * -protocol VERSION field for the detection.
+# ifdef USE_NETZIP
+ CHKiRet(uncompressMessage(pMsg));
+# endif
+
+ /* we take the risk to print a non-sanitized string, because this is the best we can get
+ * (and that functionality is too important for debugging to drop it...).
+ */
+ DBGPRINTF("msg parser: flags %x, from '%s', msg '%.60s'\n", pMsg->msgFlags,
+ (pMsg->msgFlags & NEEDS_DNSRESOL) ? UCHAR_CONSTANT("~NOTRESOLVED~") : getRcvFrom(pMsg),
+ pMsg->pszRawMsg);
+
+ /* we now need to go through our list of parsers and see which one is capable of
+ * parsing the message. Note that the first parser that requires message sanitization
+ * will cause it to happen. After that, access to the unsanitized message is no
+ * loger possible.
*/
- if(msg[0] == '1' && msg[1] == ' ') {
- dbgprintf("Message has syslog-protocol format.\n");
- setProtocolVersion(pMsg, 1);
- if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) {
- msgDestruct(&pMsg);
- ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases!
+ pParserList = ruleset.GetParserList(pMsg);
+ if(pParserList == NULL) {
+ pParserList = pDfltParsLst;
+ }
+ DBGPRINTF("parse using parser list %p%s.\n", pParserList,
+ (pParserList == pDfltParsLst) ? " (the default list)" : "");
+
+ bIsSanitized = FALSE;
+ bPRIisParsed = FALSE;
+ while(pParserList != NULL) {
+ pParser = pParserList->pParser;
+ if(pParser->bDoSanitazion && bIsSanitized == FALSE) {
+ CHKiRet(SanitizeMsg(pMsg));
+ if(pParser->bDoPRIParsing && bPRIisParsed == FALSE) {
+ CHKiRet(ParsePRI(pMsg));
+ bPRIisParsed = TRUE;
+ }
+ bIsSanitized = TRUE;
}
- } else { /* we have legacy syslog */
- dbgprintf("Message has legacy syslog format.\n");
- setProtocolVersion(pMsg, 0);
- if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) {
- msgDestruct(&pMsg);
- ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases!
+ localRet = pParser->pModule->mod.pm.parse(pMsg);
+ dbgprintf("Parser '%s' returned %d\n", pParser->pName, localRet);
+ if(localRet != RS_RET_COULD_NOT_PARSE)
+ break;
+ pParserList = pParserList->pNext;
+ }
+
+ /* We need to log a warning message and drop the message if we did not find a parser.
+ * Note that we log at most the first 1000 message, as this may very well be a problem
+ * that causes a message generation loop. We do not synchronize that counter, it doesn't
+ * matter if we log a handful messages more than we should...
+ */
+ if(localRet != RS_RET_OK) {
+ if(++iErrMsgRateLimiter > 1000) {
+ errmsg.LogError(0, localRet, "Error: one message could not be processed by "
+ "any parser, message is being discarded (start of raw msg: '%.50s')",
+ pMsg->pszRawMsg);
}
+ DBGPRINTF("No parser could process the message (state %d), we need to discard it.\n", localRet);
+ ABORT_FINALIZE(localRet);
}
- /* finalize message object */
+ /* "finalize" message object */
pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */
finalize_it:
RETiRet;
}
+
+/* set the parser name - string is copied over, call can continue to use it,
+ * but must free it if desired.
+ */
+static rsRetVal
+SetName(parser_t *pThis, uchar *name)
+{
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, parser);
+ assert(name != NULL);
+
+ if(pThis->pName != NULL) {
+ free(pThis->pName);
+ pThis->pName = NULL;
+ }
+
+ CHKmalloc(pThis->pName = ustrdup(name));
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* set a pointer to "our" module. Note that no module
+ * pointer must already be set.
+ */
+static rsRetVal
+SetModPtr(parser_t *pThis, modInfo_t *pMod)
+{
+ ISOBJ_TYPE_assert(pThis, parser);
+ assert(pMod != NULL);
+ assert(pThis->pModule == NULL);
+ pThis->pModule = pMod;
+ return RS_RET_OK;
+}
+
+
+/* Specify if we should do standard message sanitazion before we pass the data
+ * down to the parser.
+ */
+static rsRetVal
+SetDoSanitazion(parser_t *pThis, int bDoIt)
+{
+ ISOBJ_TYPE_assert(pThis, parser);
+ pThis->bDoSanitazion = bDoIt;
+ return RS_RET_OK;
+}
+
+
+/* Specify if we should do standard PRI parsing before we pass the data
+ * down to the parser module.
+ */
+static rsRetVal
+SetDoPRIParsing(parser_t *pThis, int bDoIt)
+{
+ ISOBJ_TYPE_assert(pThis, parser);
+ pThis->bDoPRIParsing = bDoIt;
+ return RS_RET_OK;
+}
+
+
+/* queryInterface function-- rgerhards, 2009-11-03
+ */
+BEGINobjQueryInterface(parser)
+CODESTARTobjQueryInterface(parser)
+ if(pIf->ifVersion != parserCURR_IF_VERSION) { /* check for current version, increment on each change */
+ ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED);
+ }
+
+ /* ok, we have the right interface, so let's fill it
+ * Please note that we may also do some backwards-compatibility
+ * work here (if we can support an older interface version - that,
+ * of course, also affects the "if" above).
+ */
+ pIf->Construct = parserConstruct;
+ pIf->ConstructFinalize = parserConstructFinalize;
+ pIf->Destruct = parserDestruct;
+ pIf->SetName = SetName;
+ pIf->SetModPtr = SetModPtr;
+ pIf->SetDoSanitazion = SetDoSanitazion;
+ pIf->SetDoPRIParsing = SetDoPRIParsing;
+ pIf->ParseMsg = ParseMsg;
+ pIf->SanitizeMsg = SanitizeMsg;
+ pIf->InitParserList = InitParserList;
+ pIf->DestructParserList = DestructParserList;
+ pIf->AddParserToList = AddParserToList;
+ pIf->AddDfltParser = AddDfltParser;
+ pIf->FindParser = FindParser;
+finalize_it:
+ENDobjQueryInterface(parser)
+
+
+
+/* Reset config variables to default values.
+ * rgerhards, 2007-07-17
+ */
+static rsRetVal
+resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal)
+{
+ cCCEscapeChar = '#';
+ bEscapeCCOnRcv = 1; /* default is to escape control characters */
+ bEscape8BitChars = 0; /* default is to escape control characters */
+ bEscapeTab = 1; /* default is to escape control characters */
+ bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */
+
+ return RS_RET_OK;
+}
+
+/* This destroys the master parserlist and all of its parser entries. MUST only be
+ * done when the module is shut down. Parser modules are NOT unloaded, rsyslog
+ * does that at a later stage for all dynamically loaded modules.
+ */
+static void
+destroyMasterParserList(void)
+{
+ parserList_t *pParsLst;
+ parserList_t *pParsLstDel;
+
+ pParsLst = pParsLstRoot;
+ while(pParsLst != NULL) {
+ parserDestruct(&pParsLst->pParser);
+ pParsLstDel = pParsLst;
+ pParsLst = pParsLst->pNext;
+ free(pParsLstDel);
+ }
+}
+
+/* Exit our class.
+ * rgerhards, 2009-11-04
+ */
+BEGINObjClassExit(parser, OBJ_IS_CORE_MODULE) /* class, version */
+ DestructParserList(&pDfltParsLst);
+ destroyMasterParserList();
+ objRelease(glbl, CORE_COMPONENT);
+ objRelease(errmsg, CORE_COMPONENT);
+ objRelease(datetime, CORE_COMPONENT);
+ objRelease(ruleset, CORE_COMPONENT);
+ENDObjClassExit(parser)
+
+
+/* Initialize the parser class. Must be called as the very first method
+ * before anything else is called inside this class.
+ * rgerhards, 2009-11-02
+ */
+BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */
+ /* request objects we use */
+ CHKiRet(objUse(glbl, CORE_COMPONENT));
+ CHKiRet(objUse(errmsg, CORE_COMPONENT));
+ CHKiRet(objUse(datetime, CORE_COMPONENT));
+ CHKiRet(objUse(ruleset, CORE_COMPONENT));
+
+ CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL));
+ CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL));
+ CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL));
+ CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL));
+ CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL));
+ CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL));
+
+ InitParserList(&pParsLstRoot);
+ InitParserList(&pDfltParsLst);
+ENDObjClassInit(parser)
+
diff --git a/runtime/parser.h b/runtime/parser.h
index cec9c083..bdd572cb 100644
--- a/runtime/parser.h
+++ b/runtime/parser.h
@@ -1,8 +1,6 @@
/* header for parser.c
- * This is not yet an object, but contains all those code necessary to
- * parse syslog messages.
*
- * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -21,10 +19,53 @@
*
* A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
*/
-#ifndef INCLUDED_PARSE_H
-#define INCLUDED_PARSE_H
+#ifndef INCLUDED_PARSER_H
+#define INCLUDED_PARSER_H
-extern rsRetVal parserClassInit(void);
-extern rsRetVal parseMsg(msg_t*);
-#endif /* #ifndef INCLUDED_PARSE_H */
+/* we create a small helper object, a list of parsers, that we can use to
+ * build a chain of them whereever this is needed (initially thought to be
+ * used in ruleset.c as well as ourselvs).
+ */
+struct parserList_s {
+ parser_t *pParser;
+ parserList_t *pNext;
+};
+
+
+/* the parser object, a dummy because we have only static methods */
+struct parser_s {
+ BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */
+ uchar *pName; /* name of this parser */
+ modInfo_t *pModule; /* pointer to parser's module */
+ sbool bDoSanitazion; /* do standard message sanitazion before calling parser? */
+ sbool bDoPRIParsing; /* do standard PRI parsing before calling parser? */
+};
+
+/* interfaces */
+BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */
+ INTERFACEObjDebugPrint(var);
+ rsRetVal (*Construct)(parser_t **ppThis);
+ rsRetVal (*ConstructFinalize)(parser_t *pThis);
+ rsRetVal (*Destruct)(parser_t **ppThis);
+ rsRetVal (*SetName)(parser_t *pThis, uchar *name);
+ rsRetVal (*SetModPtr)(parser_t *pThis, modInfo_t *pMod);
+ rsRetVal (*SetDoSanitazion)(parser_t *pThis, int);
+ rsRetVal (*SetDoPRIParsing)(parser_t *pThis, int);
+ rsRetVal (*FindParser)(parser_t **ppThis, uchar*name);
+ rsRetVal (*InitParserList)(parserList_t **pListRoot);
+ rsRetVal (*DestructParserList)(parserList_t **pListRoot);
+ rsRetVal (*AddParserToList)(parserList_t **pListRoot, parser_t *pParser);
+ /* static functions */
+ rsRetVal (*ParseMsg)(msg_t *pMsg);
+ rsRetVal (*SanitizeMsg)(msg_t *pMsg);
+ rsRetVal (*AddDfltParser)(uchar *);
+ENDinterface(parser)
+#define parserCURR_IF_VERSION 1 /* increment whenever you change the interface above! */
+
+
+/* prototypes */
+PROTOTYPEObj(parser);
+
+
+#endif /* #ifndef INCLUDED_PARSER_H */
diff --git a/runtime/prop.c b/runtime/prop.c
index 7f2a56ff..d925bb43 100644
--- a/runtime/prop.c
+++ b/runtime/prop.c
@@ -85,7 +85,7 @@ static rsRetVal SetString(prop_t *pThis, uchar *psz, int len)
if(len < CONF_PROP_BUFSIZE) {
memcpy(pThis->szVal.sz, psz, len + 1);
} else {
- CHKmalloc(pThis->szVal.psz = malloc(len + 1));
+ CHKmalloc(pThis->szVal.psz = MALLOC(len + 1));
memcpy(pThis->szVal.psz, psz, len + 1);
}
diff --git a/runtime/queue.c b/runtime/queue.c
index 9c7f96d0..9012abeb 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -8,7 +8,11 @@
* (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it
* if you are getting aquainted to the object.
*
- * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ * NOTE: as of 2009-04-22, I have begin to remove the qqueue* prefix from static
+ * function names - this makes it really hard to read and does not provide much
+ * benefit, at least I (now) think so...
+ *
+ * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -51,71 +55,186 @@
#include "wti.h"
#include "msg.h"
#include "atomic.h"
+#include "errmsg.h"
+#include "datetime.h"
+#include "unicode-helper.h"
+#include "statsobj.h"
+#include "msg.h" /* TODO: remove once we remove MsgAddRef() call */
#ifdef OS_SOLARIS
# include <sched.h>
-# define pthread_yield() sched_yield()
#endif
/* static data */
DEFobjStaticHelpers
DEFobjCurrIf(glbl)
DEFobjCurrIf(strm)
+DEFobjCurrIf(errmsg)
+DEFobjCurrIf(datetime)
+DEFobjCurrIf(statsobj)
/* forward-definitions */
-static rsRetVal qqueueChkPersist(qqueue_t *pThis);
-static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex);
-static rsRetVal qqueueRateLimiter(qqueue_t *pThis);
+static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr);
+static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates);
+static rsRetVal RateLimiter(qqueue_t *pThis);
static int qqueueChkStopWrkrDA(qqueue_t *pThis);
-static int qqueueIsIdleDA(qqueue_t *pThis);
-static rsRetVal qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave);
-static rsRetVal qqueueConsumerCancelCleanup(void *arg1, void *arg2);
-static rsRetVal qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex);
+static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal);
+static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti);
+static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti);
+static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);
+static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);
/* some constants for queuePersist () */
#define QUEUE_CHECKPOINT 1
#define QUEUE_NO_CHECKPOINT 0
+/* debug aid */
+static void displayBatchState(batch_t *pBatch)
+{
+ int i;
+ for(i = 0 ; i < pBatch->nElem ; ++i) {
+ dbgprintf("XXXXX: displayBatchState %p[%d]: %d\n", pBatch, i, pBatch->pElem[i].state);
+ }
+}
+
+/***********************************************************************
+ * we need a private data structure, the "to-delete" list. As C does
+ * not provide any partly private data structures, we implement this
+ * structure right here inside the module.
+ * Note that this list must always be kept sorted based on a unique
+ * dequeue ID (which is monotonically increasing).
+ * rgerhards, 2009-05-18
+ ***********************************************************************/
+
+/* generate next uniqueue dequeue ID. Note that uniqueness is only required
+ * on a per-queue basis and while this instance runs. So a stricly monotonically
+ * increasing counter is sufficient (if enough bits are used).
+ */
+static inline qDeqID getNextDeqID(qqueue_t *pQueue)
+{
+ ISOBJ_TYPE_assert(pQueue, qqueue);
+ return pQueue->deqIDAdd++;
+}
+
+
+/* return the top element of the to-delete list or NULL, if the
+ * list is empty.
+ */
+static inline toDeleteLst_t *tdlPeek(qqueue_t *pQueue)
+{
+ ISOBJ_TYPE_assert(pQueue, qqueue);
+ return pQueue->toDeleteLst;
+}
+
+
+/* remove the top element of the to-delete list. Nothing but the
+ * element itself is destroyed. Must not be called when the list
+ * is empty.
+ */
+static inline rsRetVal tdlPop(qqueue_t *pQueue)
+{
+ toDeleteLst_t *pRemove;
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pQueue, qqueue);
+ assert(pQueue->toDeleteLst != NULL);
+
+ pRemove = pQueue->toDeleteLst;
+ pQueue->toDeleteLst = pQueue->toDeleteLst->pNext;
+ free(pRemove);
+
+ RETiRet;
+}
+
+
+/* Add a new to-delete list entry. The function allocates the data
+ * structure, populates it with the values provided and links the new
+ * element into the correct place inside the list.
+ */
+static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElemDeq)
+{
+ toDeleteLst_t *pNew;
+ toDeleteLst_t *pPrev;
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pQueue, qqueue);
+ assert(pQueue->toDeleteLst != NULL);
+
+ CHKmalloc(pNew = MALLOC(sizeof(toDeleteLst_t)));
+ pNew->deqID = deqID;
+ pNew->nElemDeq = nElemDeq;
+
+ /* now find right spot */
+ for( pPrev = pQueue->toDeleteLst
+ ; pPrev != NULL && deqID > pPrev->deqID
+ ; pPrev = pPrev->pNext) {
+ /*JUST SEARCH*/;
+ }
+
+ if(pPrev == NULL) {
+ pNew->pNext = pQueue->toDeleteLst;
+ pQueue->toDeleteLst = pNew;
+ } else {
+ pNew->pNext = pPrev->pNext;
+ pPrev->pNext = pNew;
+ }
+
+finalize_it:
+ RETiRet;
+}
+
+
/* methods */
-/* get the overall queue size, which includes ungotten objects. Must only be called
+/* get the physical queue size. Must only be called
* while mutex is locked!
* rgerhards, 2008-01-29
*/
static inline int
-qqueueGetOverallQueueSize(qqueue_t *pThis)
+getPhysicalQueueSize(qqueue_t *pThis)
{
-#if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */
-BEGINfunc
-dbgoprint((obj_t*) pThis, "queue size: %d (regular %d, ungotten %d)\n",
- pThis->iQueueSize + pThis->iUngottenObjs, pThis->iQueueSize, pThis->iUngottenObjs);
-ENDfunc
-#endif
- return pThis->iQueueSize + pThis->iUngottenObjs;
+ return pThis->iQueueSize;
+}
+
+
+/* get the logical queue size (that is store size minus logically dequeued elements).
+ * Must only be called while mutex is locked!
+ * rgerhards, 2009-05-19
+ */
+static inline int
+getLogicalQueueSize(qqueue_t *pThis)
+{
+ return pThis->iQueueSize - pThis->nLogDeq;
}
+
/* This function drains the queue in cases where this needs to be done. The most probable
* reason is a HUP which needs to discard data (because the queue is configured to be lossy).
* During a shutdown, this is typically not needed, as the OS frees up ressources and does
* this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21
* This function returns void, as it makes no sense to communicate an error back, even if
* it happens.
+ * This functions works "around" the regular deque mechanism, because it is only used to
+ * clean up (in cases where message loss is acceptable).
*/
static inline void queueDrain(qqueue_t *pThis)
{
void *pUsr;
-
ASSERT(pThis != NULL);
+ BEGINfunc
+ DBGOPRINT((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize);
/* iQueueSize is not decremented by qDel(), so we need to do it ourselves */
while(ATOMIC_DEC_AND_FETCH(&pThis->iQueueSize, &pThis->mutQueueSize) > 0) {
- pThis->qDel(pThis, &pUsr);
+ pThis->qDeq(pThis, &pUsr);
if(pUsr != NULL) {
objDestruct(pUsr);
}
+ pThis->qDel(pThis);
}
+ ENDfunc
}
@@ -126,7 +245,8 @@ static inline void queueDrain(qqueue_t *pThis)
* this point in time. The mutex must be locked when
* ths function is called. -- rgerhards, 2008-01-25
*/
-static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis)
+static inline rsRetVal
+qqueueAdviseMaxWorkers(qqueue_t *pThis)
{
DEFiRet;
int iMaxWorkers;
@@ -134,96 +254,18 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis)
ISOBJ_TYPE_assert(pThis, qqueue);
if(!pThis->bEnqOnly) {
- if(pThis->bRunsDA) {
- /* if we have not yet reached the high water mark, there is no need to start a
- * worker. -- rgerhards, 2008-01-26
- */
- if(qqueueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) {
- wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */
- }
+ if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) {
+ DBGOPRINT((obj_t*) pThis, "(re)activating DA worker\n");
+ wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */
} else {
- if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) {
+ if(getLogicalQueueSize(pThis) == 0) {
+ iMaxWorkers = 0;
+ } else if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) {
iMaxWorkers = 1;
} else {
- iMaxWorkers = qqueueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1;
+ iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1;
}
- wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */
- }
- }
-
- RETiRet;
-}
-
-
-/* wait until we have a fully initialized DA queue. Sometimes, we need to
- * sync with it, as we expect it for some function.
- * rgerhards, 2008-02-27
- */
-static rsRetVal
-qqueueWaitDAModeInitialized(qqueue_t *pThis)
-{
- DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
- ASSERT(pThis->bRunsDA);
-
- while(pThis->bRunsDA != 2) {
- d_pthread_cond_wait(&pThis->condDAReady, pThis->mut);
- }
-
- RETiRet;
-}
-
-
-/* Destruct DA queue. This is the last part of DA-to-normal-mode
- * transistion. This is called asynchronously and some time quite a
- * while after the actual transistion. The key point is that we need to
- * do it at some later time, because we need to destruct the DA queue. That,
- * however, can not be done in a thread that has been signalled
- * This is to be called when we revert back to our own queue.
- * This function must be called with the queue mutex locked (the wti
- * class ensures this).
- * rgerhards, 2008-01-15
- */
-static rsRetVal
-qqueueTurnOffDAMode(qqueue_t *pThis)
-{
- DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
- ASSERT(pThis->bRunsDA);
-
- /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need
- * to wait for its startup... -- rgerhards, 2008-01-25
- */
- qqueueWaitDAModeInitialized(pThis);
-
- /* if we need to pull any data that we still need from the (child) disk queue,
- * now would be the time to do so. At present, we do not need this, but I'd like to
- * keep that comment if future need arises.
- */
-
- /* we need to check if the DA queue is empty because the DA worker may simply have
- * terminated do to no new messages arriving. That does not, however, mean that the
- * DA queue is empty. If there is still data in that queue, we do nothing and leave
- * that for a later incarnation of this function (it will be called multiple times
- * during the lifetime of DA-mode, depending on how often the DA worker receives an
- * inactivity timeout. -- rgerhards, 2008-01-25
- */
- if(pThis->pqDA->iQueueSize == 0) {
- pThis->bRunsDA = 0; /* tell the world we are back in non-DA mode */
- /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty,
- * this will be quick.
- */
- qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */
- dbgoprint((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n",
- iRet);
- /* now we need to check if the regular queue has some messages. This may be the case
- * when it is waiting that the high water mark is reached again. If so, we need to start up
- * a regular worker. -- rgerhards, 2008-01-26
- */
- if(qqueueGetOverallQueueSize(pThis) > 0) {
- qqueueAdviseMaxWorkers(pThis);
+ wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers);
}
}
@@ -246,37 +288,26 @@ qqueueChkIsDA(qqueue_t *pThis)
ISOBJ_TYPE_assert(pThis, qqueue);
if(pThis->pszFilePrefix != NULL) {
pThis->bIsDA = 1;
- dbgoprint((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n");
+ DBGOPRINT((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n");
} else {
- dbgoprint((obj_t*) pThis, "is NOT disk-assisted\n");
+ DBGOPRINT((obj_t*) pThis, "is NOT disk-assisted\n");
}
RETiRet;
}
-/* Start disk-assisted queue mode. All internal settings are changed. This is supposed
- * to be called from the DA worker, which must have been started before. The most important
- * chore of this function is to create the DA queue object. If that function fails,
- * the DA worker should return with an appropriate state, which in turn should lead to
- * a re-set to non-DA mode in the Enq process. The queue mutex must be locked when this
- * function is called, else a number of races will happen.
- * Please note that this function may be called *while* we in DA mode. This is due to the
- * fact that the DA worker calls it and the DA worker may be suspended (and restarted) due
- * to inactivity timeouts.
+/* Start disk-assisted queue mode.
* rgerhards, 2008-01-15
*/
static rsRetVal
-qqueueStartDA(qqueue_t *pThis)
+StartDA(qqueue_t *pThis)
{
DEFiRet;
uchar pszDAQName[128];
ISOBJ_TYPE_assert(pThis, qqueue);
- if(pThis->bRunsDA == 2) /* check if already in (fully initialized) DA mode... */
- FINALIZE; /* ... then we are already done! */
-
/* create message queue */
CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer));
@@ -298,39 +329,22 @@ qqueueStartDA(qqueue_t *pThis)
CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles));
CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown));
CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq));
- CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED));
CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr));
CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr));
+ CHKiRet(qqueueSettoQShutdown(pThis->pqDA, pThis->toQShutdown));
CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0));
CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0));
- if(pThis->toQShutdown == 0) {
- CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */
- } else {
- /* we use the shortest possible shutdown (0 is endless!) because when we run on disk AND
- * have an obviously large backlog, we can't finish it in any case. So there is no point
- * in holding shutdown longer than necessary. -- rgerhards, 2008-01-15
- */
- CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 1));
- }
iRet = qqueueStart(pThis->pqDA);
/* file not found is expected, that means it is no previous QIF available */
- if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND)
+ if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) {
+ errno = 0; /* else an errno is shown in errmsg! */
+ errmsg.LogError(errno, iRet, "error starting up disk queue, using pure in-memory mode");
+ pThis->bIsDA = 0; /* disable memory mode */
FINALIZE; /* something is wrong */
+ }
- /* as we are right now starting DA mode because we are so busy, it is
- * extremely unlikely that any regular worker is sleeping on empty queue. HOWEVER,
- * we want to be on the safe side, and so we awake anyone that is waiting
- * on one. So even if the scheduler plays badly with us, things should be
- * quite well. -- rgerhards, 2008-01-15
- */
- wtpWakeupWrkr(pThis->pWtpReg); /* awake all workers, but not ourselves ;) */
-
- pThis->bRunsDA = 2; /* we are now in DA mode, but not fully initialized */
- pThis->bChildIsDone = 0;/* set to 1 when child's worker detect queue is finished */
- pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */
-
- dbgoprint((obj_t*) pThis, "is now running in disk assisted mode, disk queue 0x%lx\n",
+ DBGOPRINT((obj_t*) pThis, "DA queue initialized, disk queue 0x%lx\n",
qqueueGetID(pThis->pqDA));
finalize_it:
@@ -338,7 +352,7 @@ finalize_it:
if(pThis->pqDA != NULL) {
qqueueDestruct(&pThis->pqDA);
}
- dbgoprint((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet);
+ DBGOPRINT((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet);
pThis->bIsDA = 0;
}
@@ -352,8 +366,8 @@ finalize_it:
* If this function fails (should not happen), DA mode is not turned on.
* rgerhards, 2008-01-16
*/
-static inline rsRetVal
-qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex)
+static rsRetVal
+InitDA(qqueue_t *pThis, int bLockMutex)
{
DEFiRet;
DEFVARS_mutexProtection;
@@ -366,82 +380,30 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex)
* is intentional. We assume that when we need it once, we may also need it on another
* occasion. Ressources used are quite minimal when no worker is running.
* rgerhards, 2008-01-24
+ * NOTE: this is the DA worker *pool*, not the DA queue!
*/
- if(pThis->pWtpDA == NULL) {
- lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DA", obj.GetName((obj_t*) pThis));
- CHKiRet(wtpConstruct (&pThis->pWtpDA));
- CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf));
- CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA));
- CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA));
- CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerDA));
- CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) qqueueConsumerCancelCleanup));
- CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA));
- CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode));
- CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut));
- CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty));
- CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1));
- CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown));
- CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis));
- CHKiRet(wtpConstructFinalize (pThis->pWtpDA));
- }
+ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis));
+ CHKiRet(wtpConstruct (&pThis->pWtpDA));
+ CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf));
+ CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA));
+ CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize));
+ CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerDA));
+ CHKiRet(wtpSetpfObjProcessed (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed));
+ CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut));
+ CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty));
+ CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1));
+ CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown));
+ CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis));
+ CHKiRet(wtpConstructFinalize (pThis->pWtpDA));
/* if we reach this point, we have a "good" DA worker pool */
- /* indicate we now run in DA mode - this is reset by the DA worker if it fails */
- pThis->bRunsDA = 1;
- pThis->bDAEnqOnly = bEnqOnly;
-
- /* now we must now adivse the wtp that we need one worker. If none is yet active,
- * that will also start one up. If we forgot that step, everything would be stalled
- * until the next enqueue request.
- */
- wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues alsways have just one worker max */
-
-finalize_it:
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
- RETiRet;
-}
-
-
-/* check if we need to start disk assisted mode and send some signals to
- * keep it running if we are already in it. It also checks if DA mode is
- * partially initialized, in which case it waits for initialization to
- * complete.
- * rgerhards, 2008-01-14
- */
-static inline rsRetVal
-qqueueChkStrtDA(qqueue_t *pThis)
-{
- DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
-
- /* if we do not hit the high water mark, we have nothing to do */
- if(qqueueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk)
- ABORT_FINALIZE(RS_RET_OK);
-
- if(pThis->bRunsDA) {
- /* then we need to signal that we are at the high water mark again. If that happens
- * on our way down the queue, that doesn't matter, because then nobody is waiting
- * on the condition variable.
- * (Remember that a DA queue stops draining the queue once it has reached the low
- * water mark and restarts it when the high water mark is reached again - this is
- * what this code here is responsible for. Please note that all workers may have been
- * terminated due to the inactivity timeout, thus we need to advise the pool that
- * we need at least one).
- */
- dbgoprint((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n",
- qqueueGetOverallQueueSize(pThis));
- qqueueAdviseMaxWorkers(pThis);
- } else {
- /* this is the case when we are currently not running in DA mode. So it is time
- * to turn it back on.
- */
- dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n",
- qqueueGetOverallQueueSize(pThis));
- qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */
+ /* now construct the actual queue (if it does not already exist) */
+ if(pThis->pqDA == NULL) {
+ CHKiRet(StartDA(pThis));
}
finalize_it:
+ END_MTX_PROTECTED_OPERATIONS(pThis->mut);
RETiRet;
}
@@ -465,10 +427,11 @@ static rsRetVal qConstructFixedArray(qqueue_t *pThis)
if(pThis->iMaxQueueSize == 0)
ABORT_FINALIZE(RS_RET_QSIZE_ZERO);
- if((pThis->tVars.farray.pBuf = malloc(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) {
+ if((pThis->tVars.farray.pBuf = MALLOC(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
+ pThis->tVars.farray.deqhead = 0;
pThis->tVars.farray.head = 0;
pThis->tVars.farray.tail = 0;
@@ -486,9 +449,7 @@ static rsRetVal qDestructFixedArray(qqueue_t *pThis)
ASSERT(pThis != NULL);
queueDrain(pThis); /* discard any remaining queue entries */
-
- if(pThis->tVars.farray.pBuf != NULL)
- free(pThis->tVars.farray.pBuf);
+ free(pThis->tVars.farray.pBuf);
RETiRet;
}
@@ -507,76 +468,37 @@ static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in)
RETiRet;
}
-static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out)
+
+static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out)
{
DEFiRet;
ASSERT(pThis != NULL);
- *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.head];
+ *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.deqhead];
- pThis->tVars.farray.head++;
- if (pThis->tVars.farray.head == pThis->iMaxQueueSize)
- pThis->tVars.farray.head = 0;
+ pThis->tVars.farray.deqhead++;
+ if (pThis->tVars.farray.deqhead == pThis->iMaxQueueSize)
+ pThis->tVars.farray.deqhead = 0;
RETiRet;
}
-/* -------------------- linked list -------------------- */
-
-/* first some generic functions which are also used for the unget linked list */
-
-static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr)
+static rsRetVal qDelFixedArray(qqueue_t *pThis)
{
DEFiRet;
- qLinkedList_t *pEntry;
-
- ASSERT(ppRoot != NULL);
- ASSERT(ppLast != NULL);
-
- if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) {
- ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
- }
- pEntry->pNext = NULL;
- pEntry->pUsr = pUsr;
+ ASSERT(pThis != NULL);
- if(*ppRoot == NULL) {
- *ppRoot = *ppLast = pEntry;
- } else {
- (*ppLast)->pNext = pEntry;
- *ppLast = pEntry;
- }
+ pThis->tVars.farray.head++;
+ if (pThis->tVars.farray.head == pThis->iMaxQueueSize)
+ pThis->tVars.farray.head = 0;
-finalize_it:
RETiRet;
}
-static inline rsRetVal qqueueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr)
-{
- DEFiRet;
- qLinkedList_t *pEntry;
-
- ASSERT(ppRoot != NULL);
- ASSERT(ppLast != NULL);
- ASSERT(ppUsr != NULL);
- ASSERT(*ppRoot != NULL);
-
- pEntry = *ppRoot;
- *ppUsr = pEntry->pUsr;
-
- if(*ppRoot == *ppLast) {
- *ppRoot = NULL;
- *ppLast = NULL;
- } else {
- *ppRoot = pEntry->pNext;
- }
- free(pEntry);
-
- RETiRet;
-}
-/* end generic functions which are also used for the unget linked list */
+/* -------------------- linked list -------------------- */
static rsRetVal qConstructLinkedList(qqueue_t *pThis)
@@ -585,8 +507,9 @@ static rsRetVal qConstructLinkedList(qqueue_t *pThis)
ASSERT(pThis != NULL);
- pThis->tVars.linklist.pRoot = 0;
- pThis->tVars.linklist.pLast = 0;
+ pThis->tVars.linklist.pDeqRoot = NULL;
+ pThis->tVars.linklist.pDelRoot = NULL;
+ pThis->tVars.linklist.pLast = NULL;
qqueueChkIsDA(pThis);
@@ -609,54 +532,59 @@ static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis)
static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr)
{
- DEFiRet;
-
- iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr);
-#if 0
qLinkedList_t *pEntry;
+ DEFiRet;
- ASSERT(pThis != NULL);
- if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) {
- ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
- }
+ CHKmalloc((pEntry = (qLinkedList_t*) MALLOC(sizeof(qLinkedList_t))));
pEntry->pNext = NULL;
pEntry->pUsr = pUsr;
- if(pThis->tVars.linklist.pRoot == NULL) {
- pThis->tVars.linklist.pRoot = pThis->tVars.linklist.pLast = pEntry;
+ if(pThis->tVars.linklist.pDelRoot == NULL) {
+ pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = pEntry;
} else {
pThis->tVars.linklist.pLast->pNext = pEntry;
pThis->tVars.linklist.pLast = pEntry;
}
+ if(pThis->tVars.linklist.pDeqRoot == NULL) {
+ pThis->tVars.linklist.pDeqRoot = pEntry;
+ }
+
finalize_it:
-#endif
RETiRet;
}
-static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr)
+
+static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr)
{
- DEFiRet;
- iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr);
-#if 0
qLinkedList_t *pEntry;
+ DEFiRet;
- ASSERT(pThis != NULL);
- ASSERT(pThis->tVars.linklist.pRoot != NULL);
-
- pEntry = pThis->tVars.linklist.pRoot;
+ pEntry = pThis->tVars.linklist.pDeqRoot;
+ ISOBJ_TYPE_assert(pEntry->pUsr, msg);
*ppUsr = pEntry->pUsr;
+ pThis->tVars.linklist.pDeqRoot = pEntry->pNext;
+
+ RETiRet;
+}
+
- if(pThis->tVars.linklist.pRoot == pThis->tVars.linklist.pLast) {
- pThis->tVars.linklist.pRoot = NULL;
- pThis->tVars.linklist.pLast = NULL;
+static rsRetVal qDelLinkedList(qqueue_t *pThis)
+{
+ qLinkedList_t *pEntry;
+ DEFiRet;
+
+ pEntry = pThis->tVars.linklist.pDelRoot;
+
+ if(pThis->tVars.linklist.pDelRoot == pThis->tVars.linklist.pLast) {
+ pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = NULL;
} else {
- pThis->tVars.linklist.pRoot = pEntry->pNext;
+ pThis->tVars.linklist.pDelRoot = pEntry->pNext;
}
+
free(pEntry);
-#endif
RETiRet;
}
@@ -676,43 +604,6 @@ finalize_it:
}
-/* This method checks if we have a QIF file for the current queue (no matter of
- * queue mode). Returns RS_RET_OK if we have a QIF file or an error status otherwise.
- * rgerhards, 2008-01-15
- */
-static rsRetVal
-qqueueHaveQIF(qqueue_t *pThis)
-{
- DEFiRet;
- uchar pszQIFNam[MAXFNAME];
- struct stat stat_buf;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
-
- if(pThis->pszFilePrefix == NULL)
- ABORT_FINALIZE(RS_RET_NO_FILEPREFIX);
-
- /* Construct file name */
- snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi",
- (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix);
-
- /* check if the file exists */
- if(stat((char*) pszQIFNam, &stat_buf) == -1) {
- if(errno == ENOENT) {
- dbgoprint((obj_t*) pThis, "no .qi file found\n");
- ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
- } else {
- dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno);
- ABORT_FINALIZE(RS_RET_IO_ERROR);
- }
- }
- /* If we reach this point, we have a .qi file */
-
-finalize_it:
- RETiRet;
-}
-
-
/* The method loads the persistent queue information.
* rgerhards, 2008-01-11
*/
@@ -724,8 +615,6 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis)
uchar pszQIFNam[MAXFNAME];
size_t lenQIFNam;
struct stat stat_buf;
- int iUngottenObjs;
- obj_t *pUsr;
ISOBJ_TYPE_assert(pThis, qqueue);
@@ -736,10 +625,10 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis)
/* check if the file exists */
if(stat((char*) pszQIFNam, &stat_buf) == -1) {
if(errno == ENOENT) {
- dbgoprint((obj_t*) pThis, "clean startup, no .qi file found\n");
+ DBGOPRINT((obj_t*) pThis, "clean startup, no .qi file found\n");
ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
} else {
- dbgoprint((obj_t*) pThis, "error %d trying to access .qi file\n", errno);
+ DBGOPRINT((obj_t*) pThis, "error %d trying to access .qi file\n", errno);
ABORT_FINALIZE(RS_RET_IO_ERROR);
}
}
@@ -755,25 +644,22 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis)
/* first, we try to read the property bag for ourselfs */
CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF));
- /* then the ungotten object queue */
- iUngottenObjs = pThis->iUngottenObjs;
- pThis->iUngottenObjs = 0; /* will be incremented when we add objects! */
-
- while(iUngottenObjs > 0) {
- /* fill the queue from disk */
- CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL));
- qqueueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED);
- --iUngottenObjs; /* one less */
- }
-
- /* and now the stream objects (some order as when persisted!) */
+ /* then the stream objects (same order as when persisted!) */
CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF,
(rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis));
- CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF,
+ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF,
(rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis));
+ /* create a duplicate for the read "pointer".
+ */
+
+ CHKiRet(strm.Dup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq));
+ CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */
+ CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq));
+
CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite));
- CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pRead));
+ CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDel));
+ CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDeq));
/* OK, we could successfully read the file, so we now can request that it be
* deleted when we are done with the persisted information.
@@ -785,7 +671,7 @@ finalize_it:
strm.Destruct(&psQIF);
if(iRet != RS_RET_OK) {
- dbgoprint((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n",
+ DBGOPRINT((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n",
iRet);
}
@@ -825,18 +711,26 @@ static rsRetVal qConstructDisk(qqueue_t *pThis)
CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR));
CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite));
- CHKiRet(strm.Construct(&pThis->tVars.disk.pRead));
- CHKiRet(strm.SetbSync(pThis->tVars.disk.pRead, pThis->bSyncQueueFiles));
- CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1));
- CHKiRet(strm.SetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir())));
- CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pRead, 10000000));
- CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ));
- CHKiRet(strm.SetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR));
- CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pRead));
+ CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDeq));
+ CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0));
+ CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir())));
+ CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000));
+ CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ));
+ CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR));
+ CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq));
+ CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDel));
+ CHKiRet(strm.SetbSync(pThis->tVars.disk.pReadDel, pThis->bSyncQueueFiles));
+ CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1));
+ CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDel, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir())));
+ CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000));
+ CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ));
+ CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR));
+ CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDel));
- CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix));
- CHKiRet(strm.SetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix));
+ CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix));
+ CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDeq, pThis->pszFilePrefix, pThis->lenFilePrefix));
+ CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDel, pThis->pszFilePrefix, pThis->lenFilePrefix));
}
/* now we set (and overwrite in case of a persisted restart) some parameters which
@@ -845,7 +739,8 @@ static rsRetVal qConstructDisk(qqueue_t *pThis)
* ability to read existing queue files. -- rgerhards, 2008-01-12
*/
CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize));
- CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize));
+ CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDeq, pThis->iMaxFileSize));
+ CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDel, pThis->iMaxFileSize));
finalize_it:
RETiRet;
@@ -857,11 +752,13 @@ static rsRetVal qDestructDisk(qqueue_t *pThis)
DEFiRet;
ASSERT(pThis != NULL);
-
- if (pThis->tVars.disk.pWrite != NULL)
+
+ if(pThis->tVars.disk.pWrite != NULL)
strm.Destruct(&pThis->tVars.disk.pWrite);
- if (pThis->tVars.disk.pRead != NULL)
- strm.Destruct(&pThis->tVars.disk.pRead);
+ if(pThis->tVars.disk.pReadDeq != NULL)
+ strm.Destruct(&pThis->tVars.disk.pReadDeq);
+ if(pThis->tVars.disk.pReadDel != NULL)
+ strm.Destruct(&pThis->tVars.disk.pReadDel);
RETiRet;
}
@@ -886,23 +783,37 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr)
*/
objDestruct(pUsr);
- dbgoprint((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets\n",
- nWriteCount, pThis->tVars.disk.sizeOnDisk);
+ DBGOPRINT((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets, EnqOnly:%d\n",
+ nWriteCount, pThis->tVars.disk.sizeOnDisk, pThis->bEnqOnly);
+
+finalize_it:
+ RETiRet;
+}
+
+
+static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr)
+{
+ DEFiRet;
+
+ CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL));
finalize_it:
RETiRet;
}
-static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr)
+
+static rsRetVal qDelDisk(qqueue_t *pThis)
{
+ obj_t *pDummyObj; /* we need to deserialize it... */
DEFiRet;
int64 offsIn;
int64 offsOut;
- CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsIn));
- CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pRead, NULL, NULL));
- CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsOut));
+ CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsIn));
+ CHKiRet(obj.Deserialize(&pDummyObj, (uchar*) "msg", pThis->tVars.disk.pReadDel, NULL, NULL));
+ objDestruct(pDummyObj);
+ CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsOut));
/* This time it is a bit tricky: we free disk space only upon file deletion. So we need
* to keep track of what we have read until we get an out-offset that is lower than the
@@ -914,7 +825,7 @@ static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr)
} else {
pThis->tVars.disk.sizeOnDisk -= pThis->tVars.disk.bytesRead;
pThis->tVars.disk.bytesRead = offsOut;
- dbgoprint((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk);
+ DBGOPRINT((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk);
/* awake possibly waiting enq process */
pthread_cond_signal(&pThis->notFull); /* we hold the mutex while we are in here! */
}
@@ -923,6 +834,7 @@ finalize_it:
RETiRet;
}
+
/* -------------------- direct (no queueing) -------------------- */
static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis)
{
@@ -937,8 +849,12 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis)
static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr)
{
+ batch_t singleBatch;
+ batch_obj_t batchObj;
+ int i;
DEFiRet;
+ //TODO: init batchObj (states _OK and new fields -- CHECK)
ASSERT(pThis != NULL);
/* calling the consumer is quite different here than it is from a worker thread */
@@ -946,70 +862,56 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr)
* mode the consumer probably has a lot to convey (which get's lost in the other modes
* because they are asynchronous. But direct mode is deliberately synchronous.
* rgerhards, 2008-02-12
+ * We use our knowledge about the batch_t structure below, but without that, we
+ * pay a too-large performance toll... -- rgerhards, 2009-04-22
*/
- iRet = pThis->pConsumer(pThis->pUsr, pUsr);
+ memset(&batchObj, 0, sizeof(batch_obj_t));
+ memset(&singleBatch, 0, sizeof(batch_t));
+ batchObj.state = BATCH_STATE_RDY;
+ batchObj.pUsrp = (obj_t*) pUsr;
+ batchObj.bFilterOK = 1;
+ singleBatch.nElem = 1; /* there always is only one in direct mode */
+ singleBatch.pElem = &batchObj;
+ iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate);
+ /* delete the batch string params: TODO: create its own "class" for this */
+ for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) {
+ free(batchObj.staticActStrings[i]);
+ }
+ objDestruct(pUsr);
RETiRet;
}
-static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out)
-{
- return RS_RET_OK;
-}
-
-
-/* --------------- end type-specific handlers -------------------- */
-
-
-/* unget a user pointer that has been dequeued. This functionality is especially important
- * for consumer cancel cleanup handlers. To support it, a short list of ungotten user pointers
- * is maintened in memory.
- * rgerhards, 2008-01-20
+/* "enqueue" a batch in direct mode. This is a shortcut which saves all the overhead
+ * otherwise incured. -- rgerhards, ~2010-06-23
*/
-static rsRetVal
-qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex)
+rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch)
{
DEFiRet;
- DEFVARS_mutexProtection;
- ISOBJ_TYPE_assert(pThis, qqueue);
- ISOBJ_assert(pUsr); /* TODO: we aborted right at this place at least 3 times -- race? 2008-02-28, -03-10, -03-15
- The second time I noticed it the queue was in destruction with NO worker threads
- running. The pUsr ptr was totally off and provided no clue what it may be pointing
- at (except that it looked like the static data pool). Both times, the abort happend
- inside an action queue */
+ ASSERT(pThis != NULL);
- dbgoprint((obj_t*) pThis, "ungetting user object %s\n", obj.GetName(pUsr));
- BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex);
- iRet = qqueueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr);
- ++pThis->iUngottenObjs; /* indicate one more */
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
+ /* calling the consumer is quite different here than it is from a worker thread */
+ /* we need to provide the consumer's return value back to the caller because in direct
+ * mode the consumer probably has a lot to convey (which get's lost in the other modes
+ * because they are asynchronous. But direct mode is deliberately synchronous.
+ * rgerhards, 2008-02-12
+ * We use our knowledge about the batch_t structure below, but without that, we
+ * pay a too-large performance toll... -- rgerhards, 2009-04-22
+ */
+ iRet = pThis->pConsumer(pThis->pUsr, pBatch, &pThis->bShutdownImmediate);
RETiRet;
}
-/* dequeues a user pointer from the ungotten queue. Pointers from there should always be
- * dequeued first.
- *
- * This function must only be called when the mutex is locked!
- *
- * rgerhards, 2008-01-29
- */
-static rsRetVal
-qqueueGetUngottenObj(qqueue_t *pThis, obj_t **ppUsr)
+static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis)
{
- DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
- ASSERT(ppUsr != NULL);
+ return RS_RET_OK;
+}
- iRet = qqueueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr);
- --pThis->iUngottenObjs; /* indicate one less */
- dbgoprint((obj_t*) pThis, "dequeued ungotten user object %s\n", obj.GetName(*ppUsr));
- RETiRet;
-}
+/* --------------- end type-specific handlers -------------------- */
/* generic code to add a queue entry
@@ -1028,7 +930,8 @@ qqueueAdd(qqueue_t *pThis, void *pUsr)
if(pThis->qType != QUEUETYPE_DIRECT) {
ATOMIC_INC(&pThis->iQueueSize, &pThis->mutQueueSize);
- dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize);
+ DBGOPRINT((obj_t*) pThis, "entry added, size now log %d, phys %d entries\n",
+ getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
}
finalize_it:
@@ -1036,12 +939,10 @@ finalize_it:
}
-/* generic code to remove a queue entry
- * rgerhards, 2008-01-29: we must first see if there is any object in the
- * ungotten list and, if so, dequeue it first.
+/* generic code to dequeue a queue entry
*/
static rsRetVal
-qqueueDel(qqueue_t *pThis, void *pUsr)
+qqueueDeq(qqueue_t *pThis, void **ppUsr)
{
DEFiRet;
@@ -1052,224 +953,249 @@ qqueueDel(qqueue_t *pThis, void *pUsr)
* If we decrement, however, we may lose a message. But that is better than
* losing the whole process because it loops... -- rgerhards, 2008-01-03
*/
- if(pThis->iUngottenObjs > 0) {
- iRet = qqueueGetUngottenObj(pThis, (obj_t**) pUsr);
- } else {
- iRet = pThis->qDel(pThis, pUsr);
- ATOMIC_DEC(&pThis->iQueueSize, &pThis->mutQueueSize);
- }
+ iRet = pThis->qDeq(pThis, ppUsr);
+ ATOMIC_INC(&pThis->nLogDeq, &pThis->mutLogDeq);
- dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n",
- iRet, pThis->iQueueSize);
+// DBGOPRINT((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n",
+// getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
RETiRet;
}
-/* This function shuts down all worker threads and waits until they
- * have terminated. If they timeout, they are cancelled. Parameters have been set
- * before this function is called so that DA queues will be fully persisted to
- * disk (if configured to do so).
- * rgerhards, 2008-01-24
- * Please note that this function shuts down BOTH the parent AND the child queue
- * in DA case. This is necessary because their timeouts are tightly coupled. Most
- * importantly, the timeouts would be applied twice (or logic be extremely
- * complex) if each would have its own shutdown. The function does not self check
- * this condition - the caller must make sure it is not called with a parent.
+/* Try to shut down regular and DA queue workers, within the queue timeout
+ * period. That means processing continues as usual. This is the expected
+ * usual case, where during shutdown those messages remaining are being
+ * processed. At this point, it is acceptable that the queue can not be
+ * fully depleted, that case is handled in the next step. During this phase,
+ * we first shut down the main queue DA worker to prevent new data to arrive
+ * at the DA queue, and then we ask the regular workers of both the Regular
+ * and DA queue to try complete processing.
+ * rgerhards, 2009-10-14
*/
-static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis)
+static inline rsRetVal
+tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis)
{
- DEFiRet;
- DEFVARS_mutexProtection;
struct timespec tTimeout;
rsRetVal iRetLocal;
+ DEFiRet;
ISOBJ_TYPE_assert(pThis, qqueue);
ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */
- dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n");
+ if(pThis->bIsDA) {
+ /* We need to lock the mutex, as otherwise we may have a race that prevents
+ * us from awaking the DA worker. */
+ d_pthread_mutex_lock(pThis->mut);
- /* we reduce the low water mark in any case. This is not absolutely necessary, but
- * it is useful because we enable DA mode at several spots below and so we do not need
- * to think about the low water mark each time.
- */
- pThis->iHighWtrMrk = 1; /* if we do not do this, the DA queue will not stop! */
- pThis->iLowWtrMrk = 0;
+ /* tell regular queue DA worker to stop shuffling messages to DA queue... */
+ DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode for DA worker\n");
+ pThis->pqDA->bEnqOnly = 1;
+ wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE);
+ wtpAdviseMaxWorkers(pThis->pWtpDA, 1);
+ DBGOPRINT((obj_t*) pThis, "awoke DA worker, told it to shut down.\n");
- /* first try to shutdown the queue within the regular shutdown period */
- BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */
- if(qqueueGetOverallQueueSize(pThis) > 0) {
- if(pThis->bRunsDA) {
- /* We may have waited on the low water mark. As it may have changed, we
- * see if we reactivate the worker.
- */
- wtpAdviseMaxWorkers(pThis->pWtpDA, 1);
- }
+ /* also tell the DA queue worker to shut down, so that it already knows... */
+ wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN);
+ wtpAdviseMaxWorkers(pThis->pqDA->pWtpReg, 1); /* awake its lone worker */
+ DBGOPRINT((obj_t*) pThis, "awoke DA queue regular worker, told it to shut down when done.\n");
+
+ d_pthread_mutex_unlock(pThis->mut);
}
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
- /* Now wait for the queue's workers to shut down. Note that we run into the code even if we just found
- * out there are no active workers - that doesn't matter: the wtp knows about that and so will
- * return immediately.
- * We do not yet care about the DA worker - that will be handled down later in the process.
- * Note that we must not request shutdown right now - that may introduce a race: if the regular queue
- * still runs DA assisted and the DA worker gets scheduled first, it will terminate itself (if the DA
- * queue happens to be empty at that instant). Then the regular worker enqueues messages, what will lead
- * to a restart of the worker. Of course, everything will continue to run, but in a bit sub-optimal way
- * (from a performance point of view). So we don't do anything right now. The DA queue will continue to
- * process messages and shutdown itself in any case if there is nothing to do. So we don't loose anything
- * by not requesting shutdown now.
- * rgerhards, 2008-01-25
- */
+
/* first calculate absolute timeout - we need the absolute value here, because we need to coordinate
* shutdown of both the regular and DA queue on *the same* timeout.
*/
timeoutComp(&tTimeout, pThis->toQShutdown);
- dbgoprint((obj_t*) pThis, "trying shutdown of regular workers\n");
+ DBGOPRINT((obj_t*) pThis, "trying shutdown of regular workers\n");
iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN, &tTimeout);
if(iRetLocal == RS_RET_TIMED_OUT) {
- dbgoprint((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n");
+ DBGOPRINT((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n");
} else {
- /* OK, the regular queue is now shut down. So we can now wait for the DA queue (if running DA) */
- dbgoprint((obj_t*) pThis, "regular queue workers shut down.\n");
- BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */
- if(pThis->bRunsDA) {
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
- dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n",
- qqueueGetID(pThis->pqDA));
- /* we use the same absolute timeout as above, so we do not use more than the configured
- * timeout interval!
- */
- dbgoprint((obj_t*) pThis, "trying shutdown of DA workers\n");
- iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout);
- if(iRetLocal == RS_RET_TIMED_OUT) {
- dbgoprint((obj_t*) pThis, "shutdown timed out on DA queue (this is OK)\n");
- }
- } else {
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
- }
+ DBGOPRINT((obj_t*) pThis, "regular queue workers shut down.\n");
}
- /* when we reach this point, both queues are either empty or the regular queue shutdown timeout
- * has expired. Now we need to check if we are configured to not loose messages. If so, we need
- * to persist the queue to disk (this is only possible if the queue is DA-enabled). We must also
- * set the primary queue to SHUTDOWN_IMMEDIATE, as it shall now terminate as soon as its consumer
- * is done. This is especially important as we otherwise may interfere with queue order while the
- * DA consumer is running. -- rgerhards, 2008-01-27
- * Note: there was a note that we should not wait eternally on the DA worker if we run in
- * enqueue-only note. I have reviewed the code and think there is no need for this check. Howerver,
- * I'd like to keep this note in here should we happen to run into some related trouble.
- * rgerhards, 2008-01-28
- */
- wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); /* set primary queue to shutdown only */
-
- /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */
- if(pThis->bRunsDA)
- qqueueWaitDAModeInitialized(pThis);
-
- BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */
- /* optimize parameters for shutdown of DA-enabled queues */
- if(pThis->bIsDA && qqueueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) {
- /* switch to enqueue-only mode so that no more actions happen */
- if(pThis->bRunsDA == 0) {
- qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */
+ /* OK, the worker for the regular queue is processed, on the the DA queue regular worker. */
+ if(pThis->pqDA != NULL) {
+ DBGOPRINT((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n",
+ qqueueGetID(pThis->pqDA));
+ /* we use the same absolute timeout as above, so we do not use more than the configured
+ * timeout interval!
+ */
+ DBGOPRINT((obj_t*) pThis, "trying shutdown of regular worker of DA queue\n");
+ iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN, &tTimeout);
+ if(iRetLocal == RS_RET_TIMED_OUT) {
+ DBGOPRINT((obj_t*) pThis, "shutdown timed out on DA queue worker (this is OK)\n");
} else {
- /* TODO: RACE: we may reach this point when the DA worker has been initialized (state 1)
- * but is not yet running (state 2). In this case, pThis->pqDA is NULL! rgerhards, 2008-02-27
- */
- qqueueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */
- }
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
- /* make sure we do not timeout before we are done */
- dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, eternal timeout set\n");
- timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL);
- /* and run the primary queue's DA worker to drain the queue */
- iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout);
- if(iRetLocal != RS_RET_OK) {
- dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, "
- "continuing, but results are unpredictable\n", iRetLocal);
+ DBGOPRINT((obj_t*) pThis, "DA queue worker shut down.\n");
}
- } else {
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
}
- /* now the primary queue is either empty, persisted to disk - or set to loose messages. So we
- * can now request immediate shutdown of any remaining workers. Note that if bSaveOnShutdown was set,
- * the queue is now empty. If regular workers are still running, and try to pull the next message,
- * they will automatically terminate as there no longer is any message left to process.
- */
- BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */
- if(qqueueGetOverallQueueSize(pThis) > 0) {
- timeoutComp(&tTimeout, pThis->toActShutdown);
- if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) {
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
- dbgoprint((obj_t*) pThis, "trying immediate shutdown of regular workers\n");
- iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
- if(iRetLocal == RS_RET_TIMED_OUT) {
- dbgoprint((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and "
- "triggers cancellation)\n");
- } else if(iRetLocal != RS_RET_OK) {
- dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue "
- "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal);
- }
- /* we need to re-aquire the mutex for the next check in this case! */
- BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */
+ RETiRet;
+}
+
+
+/* Try to shut down regular and DA queue workers, within the action timeout
+ * period. This aborts processing, but at the end of the current action, in
+ * a well-defined manner. During this phase, we terminate all three worker
+ * pools, including the regular queue DA worker if it not yet has terminated.
+ * Not finishing processing all messages is OK (and expected) at this stage
+ * (they may be preserved later, depending * on bSaveOnShutdown setting).
+ * rgerhards, 2009-10-14
+ */
+static rsRetVal
+tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis)
+{
+ struct timespec tTimeout;
+ rsRetVal iRetLocal;
+ DEFiRet;
+
+RUNLOG_STR("trying to shutdown workers within Action Timeout");
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */
+
+ /* instruct workers to finish ASAP, even if still work exists */
+ DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode\n");
+ pThis->bEnqOnly = 1;
+ pThis->bShutdownImmediate = 1;
+ /* now DA queue */
+ if(pThis->bIsDA) {
+ pThis->pqDA->bEnqOnly = 1;
+ pThis->pqDA->bShutdownImmediate = 1;
+ }
+
+// TODO: make sure we have at minimum a 10ms timeout - workers deserve a chance...
+ /* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */
+ timeoutComp(&tTimeout, pThis->toActShutdown);
+ DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of regular workers (if any)\n");
+ iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
+ if(iRetLocal == RS_RET_TIMED_OUT) {
+ DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and "
+ "triggers cancellation)\n");
+ } else if(iRetLocal != RS_RET_OK) {
+ DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue "
+ "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal);
+ }
+
+ if(pThis->pqDA != NULL) {
+ /* and now the same for the DA queue */
+ DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of DA queue workers\n");
+ iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
+ if(iRetLocal == RS_RET_TIMED_OUT) {
+ DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable "
+ "and triggers cancellation)\n");
+ } else if(iRetLocal != RS_RET_OK) {
+ DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA "
+ "queue in disk save mode. Continuing, but results are unpredictable\n", iRetLocal);
}
- if(pThis->bRunsDA && wtpGetCurNumWrkr(pThis->pWtpDA, LOCK_MUTEX) > 0) {
- /* and now the same for the DA queue */
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
- dbgoprint((obj_t*) pThis, "trying immediate shutdown of DA workers\n");
- iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
- if(iRetLocal == RS_RET_TIMED_OUT) {
- dbgoprint((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable and "
- "triggers cancellation)\n");
- } else if(iRetLocal != RS_RET_OK) {
- dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA queue "
- "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal);
- }
+
+ /* and now we need to terminate the DA worker itself. We always grant it a 100ms timeout,
+ * which should be sufficient and usually not be required (it is expected to have finished
+ * long before while we were processing the queue timeout in shutdown phase 1).
+ * rgerhards, 2009-10-14
+ */
+ timeoutComp(&tTimeout, 100);
+ DBGOPRINT((obj_t*) pThis, "trying regular shutdown of main queue DA worker pool\n");
+ iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
+ if(iRetLocal == RS_RET_TIMED_OUT) {
+ DBGOPRINT((obj_t*) pThis, "shutdown timed out on main queue DA worker pool "
+ "(this is not good, but probably OK)\n");
} else {
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
+ DBGOPRINT((obj_t*) pThis, "main queue DA worker pool shut down.\n");
}
- } else {
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
}
+ RETiRet;
+}
+
+
+/* This function cancels all remaining regular workers for both the main and the DA
+ * queue.
+ * rgerhards, 2009-05-29
+ */
+static rsRetVal
+cancelWorkers(qqueue_t *pThis)
+{
+ rsRetVal iRetLocal;
+ DEFiRet;
+
/* Now queue workers should have terminated. If not, we need to cancel them as we have applied
* all timeout setting. If any worker in any queue still executes, its consumer is possibly
- * long-running and cancelling is the only way to get rid of it. Note that the
- * cancellation handler will probably re-queue a user pointer, so the queue's enqueue
- * function is still needed (what is no problem as we do not yet destroy the queue - but I
- * thought it's a good idea to mention that fact). -- rgerhards, 2008-01-25
+ * long-running and cancelling is the only way to get rid of it.
*/
- dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n");
+ DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n");
iRetLocal = wtpCancelAll(pThis->pWtpReg); /* returns immediately if all threads already have terminated */
if(iRetLocal != RS_RET_OK) {
- dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker "
+ DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker "
"threads, continuing, but results are unpredictable\n", iRetLocal);
}
-
- /* TODO: think: do we really need to do this here? Can't it happen on DA queue destruction? If we
- * disable it, we get an assertion... I think this is OK, as we need to have a certain order and
- * canceling the DA workers here ensures that order. But in any instant, we may have a look at this
- * code after we have reaced the milestone. -- rgerhards, 2008-01-27
- */
/* ... and now the DA queue, if it exists (should always be after the primary one) */
if(pThis->pqDA != NULL) {
- dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n");
+ DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n");
iRetLocal = wtpCancelAll(pThis->pqDA->pWtpReg); /* returns immediately if all threads already have terminated */
if(iRetLocal != RS_RET_OK) {
- dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker "
+ DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker "
"threads, continuing, but results are unpredictable\n", iRetLocal);
}
+
+ /* finally, we cancel the main queue's DA worker pool, if it still is running. It may be
+ * restarted later to persist the queue. But we stop it, because otherwise we get into
+ * big trouble when resetting the logical dequeue pointer. This operation can only be
+ * done when *no* worker is running. So time for a shutdown... -- rgerhards, 2009-05-28
+ */
+ DBGOPRINT((obj_t*) pThis, "checking to see if main queue DA worker pool needs to be cancelled\n");
+ wtpCancelAll(pThis->pWtpDA); /* returns immediately if all threads already have terminated */
+ }
+
+ RETiRet;
+}
+
+
+/* This function shuts down all worker threads and waits until they
+ * have terminated. If they timeout, they are cancelled.
+ * rgerhards, 2008-01-24
+ * Please note that this function shuts down BOTH the parent AND the child queue
+ * in DA case. This is necessary because their timeouts are tightly coupled. Most
+ * importantly, the timeouts would be applied twice (or logic be extremely
+ * complex) if each would have its own shutdown. The function does not self check
+ * this condition - the caller must make sure it is not called with a parent.
+ * rgerhards, 2009-05-26: we do NO longer persist the queue here if bSaveOnShutdown
+ * is set. This must be handled by the caller. Not doing that cleans up the queue
+ * shutdown considerably. Also, older engines had a potential hang condition when
+ * the DA queue was already started and the DA worker configured for infinite
+ * retries and the action was during retry processing. This was a design issue,
+ * which is solved as of now. Note that the shutdown now may take a little bit
+ * longer, because we no longer can persist the queue in parallel to waiting
+ * on worker timeouts.
+ */
+static rsRetVal
+ShutdownWorkers(qqueue_t *pThis)
+{
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */
+
+ DBGOPRINT((obj_t*) pThis, "initiating worker thread shutdown sequence\n");
+
+ CHKiRet(tryShutdownWorkersWithinQueueTimeout(pThis));
+
+ if(getPhysicalQueueSize(pThis) > 0) {
+ CHKiRet(tryShutdownWorkersWithinActionTimeout(pThis));
}
+ CHKiRet(cancelWorkers(pThis));
+
/* ... finally ... all worker threads have terminated :-)
* Well, more precisely, they *are in termination*. Some cancel cleanup handlers
- * may still be running.
+ * may still be running. Note that the main queue's DA worker may still be running.
*/
- dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", qqueueGetOverallQueueSize(pThis));
+ DBGOPRINT((obj_t*) pThis, "worker threads terminated, remaining queue size log %d, phys %d.\n",
+ getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
+finalize_it:
RETiRet;
}
@@ -1281,7 +1207,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis)
* to modify some parameters before the queue is actually started.
*/
rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads,
- int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*))
+ int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*,int*))
{
DEFiRet;
qqueue_t *pThis;
@@ -1290,9 +1216,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread
ASSERT(pConsumer != NULL);
ASSERT(iWorkerThreads >= 0);
- if((pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))) == NULL) {
- ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
- }
+ CHKmalloc(pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t)));
/* we have an object, so let's fill the properties */
objConstructSetObjInfo(pThis);
@@ -1302,14 +1226,15 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread
/* set some water marks so that we have useful defaults if none are set specifically */
pThis->iFullDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 3; /* default 97% */
pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */
-
- pThis->lenSpoolDir = strlen((char*)pThis->pszSpoolDir);
+ pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir);
pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */
pThis->iQueueSize = 0;
+ pThis->nLogDeq = 0;
pThis->iMaxQueueSize = iMaxQueueSize;
pThis->pConsumer = pConsumer;
pThis->iNumWorkerThreads = iWorkerThreads;
pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */
+ pThis->iDeqBatchSize = 8; /* conservative default, should still provide good performance */
pThis->pszFilePrefix = NULL;
pThis->qType = qType;
@@ -1320,19 +1245,25 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread
pThis->qConstruct = qConstructFixedArray;
pThis->qDestruct = qDestructFixedArray;
pThis->qAdd = qAddFixedArray;
+ pThis->qDeq = qDeqFixedArray;
pThis->qDel = qDelFixedArray;
+ pThis->MultiEnq = qqueueMultiEnqObjNonDirect;
break;
case QUEUETYPE_LINKEDLIST:
pThis->qConstruct = qConstructLinkedList;
pThis->qDestruct = qDestructLinkedList;
pThis->qAdd = qAddLinkedList;
- pThis->qDel = (rsRetVal (*)(qqueue_t*,void**)) qDelLinkedList;
+ pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList;
+ pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList;
+ pThis->MultiEnq = qqueueMultiEnqObjNonDirect;
break;
case QUEUETYPE_DISK:
pThis->qConstruct = qConstructDisk;
pThis->qDestruct = qDestructDisk;
pThis->qAdd = qAddDisk;
+ pThis->qDeq = qDeqDisk;
pThis->qDel = qDelDisk;
+ pThis->MultiEnq = qqueueMultiEnqObjNonDirect;
/* special handling */
pThis->iNumWorkerThreads = 1; /* we need exactly one worker */
break;
@@ -1341,10 +1272,12 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread
pThis->qDestruct = qDestructDirect;
pThis->qAdd = qAddDirect;
pThis->qDel = qDelDirect;
+ pThis->MultiEnq = qqueueMultiEnqObjDirect;
break;
}
INIT_ATOMIC_HELPER_MUT(pThis->mutQueueSize);
+ INIT_ATOMIC_HELPER_MUT(pThis->mutLogDeq);
finalize_it:
OBJCONSTRUCT_CHECK_SUCCESS_AND_CLEANUP
@@ -1352,40 +1285,10 @@ finalize_it:
}
-/* cancellation cleanup handler for queueWorker ()
- * Updates admin structure and frees ressources.
- * Params:
- * arg1 - user pointer (in this case a qqueue_t)
- * arg2 - user data pointer (in this case a queue data element, any object [queue's pUsr ptr!])
- * Note that arg2 may be NULL, in which case no dequeued but unprocessed pUsr exists!
- * rgerhards, 2008-01-16
- */
-static rsRetVal
-qqueueConsumerCancelCleanup(void *arg1, void *arg2)
-{
- DEFiRet;
-
- qqueue_t *pThis = (qqueue_t*) arg1;
- obj_t *pUsr = (obj_t*) arg2;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
-
- if(pUsr != NULL) {
- /* make sure the data element is not lost */
- dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n");
- CHKiRet(qqueueUngetObj(pThis, pUsr, LOCK_MUTEX));
- }
-
-finalize_it:
- RETiRet;
-}
-
-
-
/* This function checks if the provided message shall be discarded and does so, if needed.
* In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to
* provide real-time creation of spool files.
- * Note: cached copies of iQueueSize and bRunsDA are provided so that no mutex locks are required.
+ * Note: cached copies of iQueueSize is provided so that no mutex locks are required.
* The caller must have obtained them while the mutex was locked. Of course, these values may no
* longer be current, but that is OK for the discard check. At worst, the message is either processed
* or discarded when it should not have been. As discarding is in itself somewhat racy and erratic,
@@ -1395,7 +1298,7 @@ finalize_it:
* the return state!
* rgerhards, 2008-01-24
*/
-static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr)
+static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, void *pUsr)
{
DEFiRet;
rsRetVal iRetLocal;
@@ -1404,15 +1307,15 @@ static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, voi
ISOBJ_TYPE_assert(pThis, qqueue);
ISOBJ_assert(pUsr);
- if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk && bRunsDA == 0) {
+ if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk) {
iRetLocal = objGetSeverity(pUsr, &iSeverity);
if(iRetLocal == RS_RET_OK && iSeverity >= pThis->iDiscardSeverity) {
- dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n",
+ DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n",
iQueueSize, iSeverity);
objDestruct(pUsr);
ABORT_FINALIZE(RS_RET_QUEUE_FULL);
} else {
- dbgoprint((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg "
+ DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg "
"(iRet: %d, severity %d)\n", iQueueSize, iRetLocal, iSeverity);
}
}
@@ -1422,40 +1325,193 @@ finalize_it:
}
-/* dequeue the queued object for the queue consumers.
- * rgerhards, 2008-10-21
+/* Finally remove n elements from the queue store.
*/
-static rsRetVal
-qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave)
+static inline rsRetVal
+DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem)
{
+ int i;
DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+
+ /* now send delete request to storage driver */
+ for(i = 0 ; i < nElem ; ++i) {
+ pThis->qDel(pThis);
+ }
+
+ /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */
+ ATOMIC_SUB(&pThis->iQueueSize, nElem, &pThis->mutQueueSize);
+ ATOMIC_SUB(&pThis->nLogDeq, nElem, &pThis->mutLogDeq);
+dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
+ ++pThis->deqIDDel; /* one more batch dequeued */
+
+ RETiRet;
+}
+
+
+/* remove messages from the physical queue store that are fully processed. This is
+ * controlled via the to-delete list.
+ */
+static inline rsRetVal
+DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch)
+{
+ toDeleteLst_t *pTdl;
+ qDeqID deqIDDel;
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ assert(pBatch != NULL);
+
+ pTdl = tdlPeek(pThis); /* get current head element */
+ if(pTdl == NULL) { /* to-delete list empty */
+ DoDeleteBatchFromQStore(pThis, pBatch->nElem);
+ } else if(pBatch->deqID == pThis->deqIDDel) {
+ deqIDDel = pThis->deqIDDel;
+ pTdl = tdlPeek(pThis);
+ while(pTdl != NULL && deqIDDel == pTdl->deqID) {
+ DoDeleteBatchFromQStore(pThis, pTdl->nElemDeq);
+ tdlPop(pThis);
+ ++deqIDDel;
+ pTdl = tdlPeek(pThis);
+ }
+ /* old entries deleted, now delete current ones... */
+ DoDeleteBatchFromQStore(pThis, pBatch->nElem);
+ } else {
+ /* can not delete, insert into to-delete list */
+ dbgprintf("not at head of to-delete list, enqueue %d\n", (int) pBatch->deqID);
+ CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem));
+ }
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* Delete a batch of processed user objects from the queue, which includes
+ * destructing the objects themself. Any entries not marked as finally
+ * processed are enqueued again. The new enqueue is necessary because we have a
+ * rgerhards, 2009-05-13
+ */
+static inline rsRetVal
+DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch)
+{
+ int i;
void *pUsr;
+ int nEnqueued = 0;
+ rsRetVal localRet;
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ assert(pBatch != NULL);
+
+ for(i = 0 ; i < pBatch->nElem ; ++i) {
+ pUsr = pBatch->pElem[i].pUsrp;
+ if( pBatch->pElem[i].state == BATCH_STATE_RDY
+ || pBatch->pElem[i].state == BATCH_STATE_SUB) {
+dbgprintf("XXX: DeleteProcessedBatch re-enqueue %d of %d, state %d\n", i, pBatch->nElem, pBatch->pElem[i].state);
+ localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY,
+ (obj_t*)MsgAddRef((msg_t*) pUsr));
+ ++nEnqueued;
+ if(localRet != RS_RET_OK) {
+ DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet);
+ }
+ }
+ objDestruct(pUsr);
+ }
+
+ dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued);
+
+ if(nEnqueued > 0)
+ qqueueChkPersist(pThis, nEnqueued);
+
+ iRet = DeleteBatchFromQStore(pThis, pBatch);
+
+ pBatch->nElem = pBatch->nElemDeq = 0; /* reset batch */ // TODO: more fine init, new fields! 2010-06-14
+
+ RETiRet;
+}
+
+
+/* dequeue as many user pointers as are available, until we hit the configured
+ * upper limit of pointers. Note that this function also deletes all processed
+ * objects from the previous batch. However, it is perfectly valid that the
+ * previous batch contained NO objects at all. For example, this happens
+ * immediately after system startup or when a queue was exhausted and the queue
+ * worker needed to wait for new data.
+ * This must only be called when the queue mutex is LOOKED, otherwise serious
+ * malfunction will happen.
+ */
+static inline rsRetVal
+DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSize)
+{
+ int nDequeued;
+ int nDiscarded;
+ int nDeleted;
int iQueueSize;
- int bRunsDA; /* cache for early mutex release */
-
- /* dequeue element (still protected from mutex) */
- iRet = qqueueDel(pThis, &pUsr);
- qqueueChkPersist(pThis);
- iQueueSize = qqueueGetOverallQueueSize(pThis); /* cache this for after mutex release */
- bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */
-
- /* We now need to save the user pointer for the cancel cleanup handler, BUT ONLY
- * if we could successfully obtain a user pointer. Otherwise, we would bring the
- * cancel cleanup handler into big troubles (and we did ;)). Note that we can
- * NOT set the variable further below, as this may lead to an object leak. We
- * may get cancelled before we reach that part of the code, so the only
- * solution is to do it here. -- rgerhards, 2008-02-27
- */
- if(iRet == RS_RET_OK) {
- pWti->pUsrp = pUsr;
+ void *pUsr;
+ rsRetVal localRet;
+ DEFiRet;
+
+ nDeleted = pWti->batch.nElemDeq;
+ DeleteProcessedBatch(pThis, &pWti->batch);
+
+ nDequeued = nDiscarded = 0;
+ while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) {
+ CHKiRet(qqueueDeq(pThis, &pUsr));
+
+ /* check if we should discard this element */
+ localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr);
+ if(localRet == RS_RET_QUEUE_FULL) {
+ ++nDiscarded;
+ continue;
+ } else if(localRet != RS_RET_OK) {
+ ABORT_FINALIZE(localRet);
+ }
+
+ /* all well, use this element */
+ pWti->batch.pElem[nDequeued].pUsrp = pUsr;
+ pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY;
+ pWti->batch.pElem[nDequeued].bFilterOK = 1; // TODO: think again if we can handle that with more performance
+ ++nDequeued;
}
+ /* it is sufficient to persist only when the bulk of work is done */
+ qqueueChkPersist(pThis, nDequeued+nDiscarded+nDeleted);
+
+ pWti->batch.nElem = nDequeued;
+ pWti->batch.nElemDeq = nDequeued + nDiscarded;
+ pWti->batch.deqID = getNextDeqID(pThis);
+ *piRemainingQueueSize = iQueueSize;
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* dequeue the queued object for the queue consumers.
+ * rgerhards, 2008-10-21
+ * I made a radical change - we now dequeue multiple elements, and store these objects in
+ * an array of user pointers. We expect that this increases performance.
+ * rgerhards, 2009-04-22
+ */
+static rsRetVal
+DequeueConsumable(qqueue_t *pThis, wti_t *pWti)
+{
+ DEFiRet;
+ int iQueueSize = 0; /* keep the compiler happy... */
+
+ /* dequeue element batch (still protected from mutex) */
+ iRet = DequeueConsumableElements(pThis, pWti, &iQueueSize);
+
/* awake some flow-controlled sources if we can do this right now */
/* TODO: this could be done better from a performance point of view -- do it only if
* we have someone waiting for the condition (or only when we hit the watermark right
* on the nail [exact value]) -- rgerhards, 2008-03-14
+ * now that we dequeue batches of pointers, this is much less an issue...
+ * rgerhards, 2009-04-22
*/
- if(iQueueSize < pThis->iFullDlyMrk / 2) {
+ if(iQueueSize < pThis->iFullDlyMrk / 2 || glbl.GetGlobalInputTermState() == 1) {
pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk);
}
@@ -1463,37 +1519,15 @@ qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave)
pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk);
}
- /* rgerhards, 2008-09-30: I reversed the order of cond_signal und mutex_unlock
- * as of the pthreads recommendation on predictable scheduling behaviour. I don't see
- * any problems caused by this, but I add this comment in case some will be seen
- * in the next time.
- */
+ // TODO: MULTI: check physical queue size?
pthread_cond_signal(&pThis->notFull);
- d_pthread_mutex_unlock(pThis->mut);
- pthread_setcancelstate(iCancelStateSave, NULL);
/* WE ARE NO LONGER PROTECTED BY THE MUTEX */
- /* do actual processing (the lengthy part, runs in parallel)
- * If we had a problem while dequeing, we do not call the consumer,
- * but we otherwise ignore it. This is in the hopes that it will be
- * self-healing. However, this is really not a good thing.
- * rgerhards, 2008-01-03
- */
- if(iRet != RS_RET_OK)
- FINALIZE;
-
- /* we are running in normal, non-disk-assisted mode do a quick check if we need to drain the queue.
- * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to
- * provide real-time creation of spool files.
- * Note: It is OK to use the cached iQueueSize here, because it does not hurt if it is slightly wrong.
- */
- CHKiRet(qqueueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr));
-
-finalize_it:
if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) {
- dbgoprint((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things "
+ DBGOPRINT((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things "
"may happen\n", iRet);
}
+
RETiRet;
}
@@ -1536,7 +1570,7 @@ finalize_it:
* but you get the idea from the code above.
*/
static rsRetVal
-qqueueRateLimiter(qqueue_t *pThis)
+RateLimiter(qqueue_t *pThis)
{
DEFiRet;
int iDelay;
@@ -1549,7 +1583,7 @@ qqueueRateLimiter(qqueue_t *pThis)
iDelay = 0;
if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */
/* time calls are expensive, so only do them when needed */
- time(&tCurr);
+ datetime.GetTime(&tCurr);
localtime_r(&tCurr, &m);
iHrCurr = m.tm_hour;
@@ -1585,7 +1619,7 @@ qqueueRateLimiter(qqueue_t *pThis)
}
if(iDelay > 0) {
- dbgoprint((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay);
+ DBGOPRINT((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay);
srSleep(iDelay, 0);
}
@@ -1593,37 +1627,109 @@ qqueueRateLimiter(qqueue_t *pThis)
}
+/* This dequeues the next batch. Note that this function must not be
+ * cancelled, else it will leave back an inconsistent state.
+ * rgerhards, 2009-05-20
+ */
+static inline rsRetVal
+DequeueForConsumer(qqueue_t *pThis, wti_t *pWti)
+{
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ ISOBJ_TYPE_assert(pWti, wti);
+
+ CHKiRet(DequeueConsumable(pThis, pWti));
+
+ if(pWti->batch.nElem == 0)
+ ABORT_FINALIZE(RS_RET_IDLE);
+
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* This is called when a batch is processed and the worker does not
+ * ask for another batch (e.g. because it is to be terminated)
+ * Note that we must not be terminated while we delete a processed
+ * batch. Otherwise, we may not complete it, and then the cancel
+ * handler also tries to delete the batch. But then it finds some of
+ * the messages already destructed. This was a bug we have seen, especially
+ * with disk mode, where a delete takes rather long. Anyhow, the coneptual
+ * problem exists in all queue modes.
+ * rgerhards, 2009-05-27
+ */
+static rsRetVal
+batchProcessed(qqueue_t *pThis, wti_t *pWti)
+{
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ ISOBJ_TYPE_assert(pWti, wti);
+
+ int iCancelStateSave;
+ /* at this spot, we must not be cancelled */
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+ DeleteProcessedBatch(pThis, &pWti->batch);
+ qqueueChkPersist(pThis, pWti->batch.nElemDeq);
+ pthread_setcancelstate(iCancelStateSave, NULL);
+
+ RETiRet;
+}
+
/* This is the queue consumer in the regular (non-DA) case. It is
* protected by the queue mutex, but MUST release it as soon as possible.
* rgerhards, 2008-01-21
*/
static rsRetVal
-qqueueConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave)
+ConsumerReg(qqueue_t *pThis, wti_t *pWti)
{
+ int iCancelStateSave;
+ int bNeedReLock = 0; /**< do we need to lock the mutex again? */
DEFiRet;
ISOBJ_TYPE_assert(pThis, qqueue);
ISOBJ_TYPE_assert(pWti, wti);
- CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave));
- CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->pUsrp));
+ CHKiRet(DequeueForConsumer(pThis, pWti));
+
+ /* we now have a non-idle batch of work, so we can release the queue mutex and process it */
+ d_pthread_mutex_unlock(pThis->mut);
+ bNeedReLock = 1;
+
+ /* at this spot, we may be cancelled */
+ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave);
+
+ CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch, &pThis->bShutdownImmediate));
/* we now need to check if we should deliberately delay processing a bit
* and, if so, do that. -- rgerhards, 2008-01-30
*/
+//TODO: MULTIQUEUE: the following setting is no longer correct - need to think about how to do that...
if(pThis->iDeqSlowdown) {
- dbgoprint((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n",
+ DBGOPRINT((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n",
pThis->iDeqSlowdown);
srSleep(pThis->iDeqSlowdown / 1000000, pThis->iDeqSlowdown % 1000000);
}
+ /* but now cancellation is no longer permitted */
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+
finalize_it:
+ DBGPRINTF("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet,
+ getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
+
+ /* now we are done, but potentially need to re-aquire the mutex */
+ if(bNeedReLock)
+ d_pthread_mutex_lock(pThis->mut);
+
RETiRet;
}
-/* This is a special consumer to feed the disk-queue in disk-assited mode.
+/* This is a special consumer to feed the disk-queue in disk-assisted mode.
* When active, our own queue more or less acts as a memory buffer to the disk.
* So this consumer just needs to drain the memory queue and submit entries
* to the disk queue. The disk queue will then call the actual consumer from
@@ -1633,151 +1739,96 @@ finalize_it:
* rgerhards, 2008-01-14
*/
static rsRetVal
-qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave)
+ConsumerDA(qqueue_t *pThis, wti_t *pWti)
{
+ int i;
+ int iCancelStateSave;
DEFiRet;
ISOBJ_TYPE_assert(pThis, qqueue);
ISOBJ_TYPE_assert(pWti, wti);
- CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave));
- CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp));
+ CHKiRet(DequeueForConsumer(pThis, pWti));
+
+ /* we now have a non-idle batch of work, so we can release the queue mutex and process it */
+ d_pthread_mutex_unlock(pThis->mut);
+
+ /* at this spot, we may be cancelled */
+ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave);
+
+ /* iterate over returned results and enqueue them in DA queue */
+ for(i = 0 ; i < pWti->batch.nElem && !pThis->bShutdownImmediate ; i++) {
+ /* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs
+ * the message. So far, we simply assume we always have msg_t, what currently is always the case.
+ * rgerhards, 2009-05-28
+ */
+ CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY,
+ (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp))));
+ pWti->batch.pElem[i].state = BATCH_STATE_COMM; /* commited to other queue! */
+ }
+
+ /* but now cancellation is no longer permitted */
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+
+ /* now we are done, but need to re-aquire the mutex */
+ d_pthread_mutex_lock(pThis->mut);
finalize_it:
- dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet);
+ DBGOPRINT((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet);
RETiRet;
}
/* must only be called when the queue mutex is locked, else results
* are not stable!
- * If we are a child, we have done our duty when the queue is empty. In that case,
- * we can terminate.
- * Version for the DA worker thread. NOTE: the pThis->bRunsDA is different from
- * the DA queue
*/
-static int
+static rsRetVal
qqueueChkStopWrkrDA(qqueue_t *pThis)
{
- /* if our queue is in destruction, we drain to the DA queue and so we shall not terminate
- * until we have done so.
- */
- int bStopWrkr;
-
- BEGINfunc
+ DEFiRet;
+//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis));
if(pThis->bEnqOnly) {
- bStopWrkr = 1;
- } else {
- if(pThis->bRunsDA == 2) {
- ASSERT(pThis->pqDA != NULL);
- if( pThis->pqDA->bEnqOnly
- && pThis->pqDA->sizeOnDiskMax > 0
- && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) {
- /* this queue can never grow, so we can give up... */
- bStopWrkr = 1;
- } else if(qqueueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) {
- bStopWrkr = 1;
- } else {
- bStopWrkr = 0;
- }
- } else {
- bStopWrkr = 1;
- }
+ iRet = RS_RET_TERMINATE_WHEN_IDLE;
+ }
+ if(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk) {
+ iRet = RS_RET_TERMINATE_NOW;
}
- ENDfunc
- return bStopWrkr;
+ RETiRet;
}
/* must only be called when the queue mutex is locked, else results
* are not stable!
* If we are a child, we have done our duty when the queue is empty. In that case,
- * we can terminate.
- * Version for the regular worker thread. NOTE: the pThis->bRunsDA is different from
- * the DA queue
- */
-static int
-qqueueChkStopWrkrReg(qqueue_t *pThis)
-{
- return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0);
-}
-
-
-/* must only be called when the queue mutex is locked, else results
- * are not stable! DA queue version
- */
-static int
-qqueueIsIdleDA(qqueue_t *pThis)
-{
- /* remember: iQueueSize is the DA queue size, not the main queue! */
- /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */
- return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk));
-}
-/* must only be called when the queue mutex is locked, else results
- * are not stable! Regular queue version
- */
-static int
-qqueueIsIdleReg(qqueue_t *pThis)
-{
-#if 0 /* enable for performance testing */
- int ret;
- ret = qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk);
- if(ret) fprintf(stderr, "queue is idle\n");
- return ret;
-#else
- /* regular code! */
- return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk));
-#endif
-}
-
-
-/* This function is called when a worker thread for the regular queue is shut down.
- * If we are the primary queue, this is not really interesting to us. If, however,
- * we are the DA (child) queue, that means the DA queue is empty. In that case, we
- * need to signal the parent queue's DA worker, so that it can terminate DA mode.
- * rgerhards, 2008-01-26
- * rgerhards, 2008-02-27: HOWEVER, in a shutdown condition, it may be that the parent's worker thread pool
- * has already been terminated and destructed. This *is* a legal condition and happens
- * from time to time in practice. So we need to signal only if there still is a
- * parent DA worker queue. Please keep in mind that the the parent's DA worker
- * pool is DIFFERENT from our (DA queue) regular worker pool. So when the parent's
- * pWtpDA is destructed, there can still be some of our (DAq/wtp) threads be running.
- * I am telling this, because I, too, always get confused by those...
+ * we can terminate. Version for the regular worker thread.
*/
static rsRetVal
-qqueueRegOnWrkrShutdown(qqueue_t *pThis)
+ChkStopWrkrReg(qqueue_t *pThis)
{
DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
-
- if(pThis->pqParent != NULL) {
- pThis->pqParent->bChildIsDone = 1; /* indicate we are done */
- if(pThis->pqParent->pWtpDA != NULL) { /* see comment in function header from 2008-02-27 */
- wtpAdviseMaxWorkers(pThis->pqParent->pWtpDA, 1); /* reactivate DA worker (always 1) */
- }
+ if(pThis->bEnqOnly) {
+ iRet = RS_RET_TERMINATE_NOW;
+ } else if(pThis->pqParent != NULL) {
+ iRet = RS_RET_TERMINATE_WHEN_IDLE;
}
RETiRet;
}
-/* The following function is called when a regular queue worker starts up. We need this
- * hook to indicate in the parent queue (if we are a child) that we are not done yet.
+/* return the configured "deq max at once" interval
+ * rgerhards, 2009-04-22
*/
static rsRetVal
-qqueueRegOnWrkrStartup(qqueue_t *pThis)
+GetDeqBatchSize(qqueue_t *pThis, int *pVal)
{
DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
-
- if(pThis->pqParent != NULL) {
- pThis->pqParent->bChildIsDone = 0;
- }
-
+ assert(pVal != NULL);
+ *pVal = pThis->iDeqBatchSize;
+if(pThis->pqParent != NULL) // TODO: check why we actually do this!
+ *pVal = 16;
RETiRet;
}
@@ -1785,12 +1836,13 @@ qqueueRegOnWrkrStartup(qqueue_t *pThis)
/* start up the queue - it must have been constructed and parameters defined
* before.
*/
-rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
+rsRetVal
+qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
{
DEFiRet;
- rsRetVal iRetLocal;
- int bInitialized = 0; /* is queue already initialized? */
uchar pszBuf[64];
+ int wrk;
+ uchar *qName;
size_t lenBuf;
ASSERT(pThis != NULL);
@@ -1803,16 +1855,15 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
* influenced by properties which might have been set after queueConstruct ()
*/
if(pThis->pqParent == NULL) {
- pThis->mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t));
+ pThis->mut = (pthread_mutex_t *) MALLOC (sizeof (pthread_mutex_t));
pthread_mutex_init(pThis->mut, NULL);
} else {
/* child queue, we need to use parent's mutex */
- dbgoprint((obj_t*) pThis, "I am a child\n");
+ DBGOPRINT((obj_t*) pThis, "I am a child\n");
pThis->mut = pThis->pqParent->mut;
}
pthread_mutex_init(&pThis->mutThrdMgmt, NULL);
- pthread_cond_init (&pThis->condDAReady, NULL);
pthread_cond_init (&pThis->notFull, NULL);
pthread_cond_init (&pThis->notEmpty, NULL);
pthread_cond_init (&pThis->belowFullDlyWtrMrk, NULL);
@@ -1821,28 +1872,36 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
/* call type-specific constructor */
CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */
- dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d, "
- "full delay %d, light delay %d starting\n",
+ /* re-adjust some params if required */
+ if(pThis->bIsDA) {
+ /* if we are in DA mode, we must make sure full delayable messages do not
+ * initiate going to disk!
+ */
+ wrk = pThis->iHighWtrMrk - (pThis->iHighWtrMrk / 100) * 50; /* 50% of high water mark */
+ if(wrk < pThis->iFullDlyMrk)
+ pThis->iFullDlyMrk = wrk;
+ }
+
+ DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, "
+ "full delay %d, light delay %d, deq batch size %d starting\n",
pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize,
- qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1,
- pThis->iFullDlyMrk, pThis->iLightDlyMrk);
+ getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis),
+ pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk,
+ pThis->iDeqBatchSize);
if(pThis->qType == QUEUETYPE_DIRECT)
FINALIZE; /* with direct queues, we are already finished... */
- /* create worker thread pools for regular operation. The DA pool is created on an as-needed
- * basis, which potentially means never under most circumstances.
+ /* create worker thread pools for regular and DA operation.
*/
lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis));
CHKiRet(wtpConstruct (&pThis->pWtpReg));
CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf));
- CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRateLimiter));
- CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrReg));
- CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleReg));
- CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerReg));
- CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))qqueueConsumerCancelCleanup));
- CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrStartup));
- CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrShutdown));
+ CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter));
+ CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg));
+ CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize));
+ CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerReg));
+ CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed));
CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut));
CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty));
CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads));
@@ -1850,27 +1909,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
CHKiRet(wtpSetpUsr (pThis->pWtpReg, pThis));
CHKiRet(wtpConstructFinalize (pThis->pWtpReg));
- /* initialize worker thread instances */
- if(pThis->bIsDA) {
- /* If we are disk-assisted, we need to check if there is a QIF file
- * which we need to load. -- rgerhards, 2008-01-15
- */
- iRetLocal = qqueueHaveQIF(pThis);
- if(iRetLocal == RS_RET_OK) {
- dbgoprint((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n");
- qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */
- bInitialized = 1; /* we are done */
- } else {
- /* TODO: use logerror? -- rgerhards, 2008-01-16 */
- dbgoprint((obj_t*) pThis, "error %d trying to access on-disk queue files, starting without them. "
- "Some data may be lost\n", iRetLocal);
- }
- }
+ /* set up DA system if we have a disk-assisted queue */
+ if(pThis->bIsDA)
+ InitDA(pThis, LOCK_MUTEX); /* initiate DA mode */
- if(!bInitialized) {
- dbgoprint((obj_t*) pThis, "queue starts up without (loading) any DA disk state (this is normal for the DA "
- "queue itself!)\n");
- }
+ DBGOPRINT((obj_t*) pThis, "queue finished initialization\n");
/* if the queue already contains data, we need to start the correct number of worker threads. This can be
* the case when a disk queue has been loaded. If we did not start it here, it would never start.
@@ -1878,6 +1921,27 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
qqueueAdviseMaxWorkers(pThis);
pThis->bQueueStarted = 1;
+ /* support statistics gathering */
+ qName = obj.GetName((obj_t*)pThis);
+ CHKiRet(statsobj.Construct(&pThis->statsobj));
+ CHKiRet(statsobj.SetName(pThis->statsobj, qName));
+ CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("size"),
+ ctrType_Int, &pThis->iQueueSize));
+
+ STATSCOUNTER_INIT(pThis->ctrEnqueued, pThis->mutCtrEnqueued);
+ CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("enqueued"),
+ ctrType_IntCtr, &pThis->ctrEnqueued));
+
+ STATSCOUNTER_INIT(pThis->ctrFull, pThis->mutCtrFull);
+ CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("full"),
+ ctrType_IntCtr, &pThis->ctrFull));
+
+ pThis->ctrMaxqsize = 0;
+ CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("maxqsize"),
+ ctrType_Int, &pThis->ctrMaxqsize));
+
+ CHKiRet(statsobj.ConstructFinalize(pThis->statsobj));
+
finalize_it:
RETiRet;
}
@@ -1896,12 +1960,11 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint)
strm_t *psQIF = NULL; /* Queue Info File */
uchar pszQIFNam[MAXFNAME];
size_t lenQIFNam;
- obj_t *pUsr;
ASSERT(pThis != NULL);
if(pThis->qType != QUEUETYPE_DISK) {
- if(qqueueGetOverallQueueSize(pThis) > 0) {
+ if(getPhysicalQueueSize(pThis) > 0) {
/* This error code is OK, but we will probably not implement this any time
* The reason is that persistence happens via DA queues. But I would like to
* leave the code as is, as we so have a hook in case we need one.
@@ -1912,20 +1975,20 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint)
FINALIZE; /* if the queue is empty, we are happy and done... */
}
- dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", qqueueGetOverallQueueSize(pThis));
+ DBGOPRINT((obj_t*) pThis, "persisting queue to disk, %d entries...\n", getPhysicalQueueSize(pThis));
/* Construct file name */
lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi",
(char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix);
- if((bIsCheckpoint != QUEUE_CHECKPOINT) && (qqueueGetOverallQueueSize(pThis) == 0)) {
+ if((bIsCheckpoint != QUEUE_CHECKPOINT) && (getPhysicalQueueSize(pThis) == 0)) {
if(pThis->bNeedDelQIF) {
unlink((char*)pszQIFNam);
pThis->bNeedDelQIF = 0;
}
/* indicate spool file needs to be deleted */
- if (pThis->tVars.disk.pRead != NULL)
- CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1));
+ if(pThis->tVars.disk.pReadDel != NULL) /* may be NULL if we had a startup failure! */
+ CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1));
FINALIZE; /* nothing left to do, so be happy */
}
@@ -1944,31 +2007,19 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint)
*/
CHKiRet(obj.BeginSerializePropBag(psQIF, (obj_t*) pThis));
objSerializeSCALAR(psQIF, iQueueSize, INT);
- objSerializeSCALAR(psQIF, iUngottenObjs, INT);
objSerializeSCALAR(psQIF, tVars.disk.sizeOnDisk, INT64);
objSerializeSCALAR(psQIF, tVars.disk.bytesRead, INT64);
CHKiRet(obj.EndSerialize(psQIF));
- /* now we must persist all objects on the ungotten queue - they can not go to
- * to the regular files. -- rgerhards, 2008-01-29
- */
- while(pThis->iUngottenObjs > 0) {
- CHKiRet(qqueueGetUngottenObj(pThis, &pUsr));
- CHKiRet((objSerialize(pUsr))(pUsr, psQIF));
- objDestruct(pUsr);
- }
-
/* now persist the stream info */
- if (pThis->tVars.disk.pWrite != NULL)
- CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF));
- if (pThis->tVars.disk.pRead != NULL)
- CHKiRet(strm.Serialize(pThis->tVars.disk.pRead, psQIF));
+ CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF));
+ CHKiRet(strm.Serialize(pThis->tVars.disk.pReadDel, psQIF));
/* tell the input file object that it must not delete the file on close if the queue
* is non-empty - but only if we are not during a simple checkpoint
*/
- if(bIsCheckpoint != QUEUE_CHECKPOINT && pThis->tVars.disk.pRead != NULL) {
- CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 0));
+ if(bIsCheckpoint != QUEUE_CHECKPOINT) {
+ CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 0));
}
/* we have persisted the queue object. So whenever it comes to an empty queue,
@@ -1985,36 +2036,86 @@ finalize_it:
/* check if we need to persist the current queue info. If an
- * error occurs, thus should be ignored by caller (but we still
+ * error occurs, this should be ignored by caller (but we still
* abide to our regular call interface)...
* rgerhards, 2008-01-13
+ * nUpdates is the number of updates since the last call to this function.
+ * It may be > 1 due to batches. -- rgerhards, 2009-05-12
*/
-static rsRetVal qqueueChkPersist(qqueue_t *pThis)
+static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates)
{
+ DEFiRet;
ISOBJ_TYPE_assert(pThis, qqueue);
+ assert(nUpdates >= 0);
- if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) {
+ if(nUpdates == 0)
+ FINALIZE;
+
+ pThis->iUpdsSincePersist += nUpdates;
+ if(pThis->iPersistUpdCnt && pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) {
qqueuePersist(pThis, QUEUE_CHECKPOINT);
pThis->iUpdsSincePersist = 0;
}
- return RS_RET_OK;
+finalize_it:
+ RETiRet;
+}
+
+
+/* persist a queue with all data elements to disk - this is used to handle
+ * bSaveOnShutdown. We utilize the DA worker to do this. This must only
+ * be called after all workers have been shut down and if bSaveOnShutdown
+ * is actually set. Note that this function may potentially run long,
+ * depending on the queue configuration (e.g. store on remote machine).
+ * rgerhards, 2009-05-26
+ */
+static inline rsRetVal
+DoSaveOnShutdown(qqueue_t *pThis)
+{
+ struct timespec tTimeout;
+ rsRetVal iRetLocal;
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+
+ /* we reduce the low water mark, otherwise the DA worker would terminate when
+ * it is reached.
+ */
+ DBGOPRINT((obj_t*) pThis, "bSaveOnShutdown set, restarting DA worker...\n");
+ pThis->bShutdownImmediate = 0; /* would termiante the DA worker! */
+ pThis->iLowWtrMrk = 0;
+ wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN); /* shutdown worker (only) when done (was _IMMEDIATE!) */
+ wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* restart DA worker */
+
+ DBGOPRINT((obj_t*) pThis, "waiting for DA worker to terminate...\n");
+ timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL);
+ /* and run the primary queue's DA worker to drain the queue */
+ iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout);
+ DBGOPRINT((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n",
+ iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
+ if(iRetLocal != RS_RET_OK) {
+ DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, "
+ "continuing, but results are unpredictable\n", iRetLocal);
+ }
+
+ RETiRet;
}
/* destructor for the queue object */
BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */
CODESTARTobjDestruct(qqueue)
- pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */
-
- /* shut down all workers (handles *all* of the persistence logic)
- * See function head comment of queueShutdownWorkers () on why we don't call it
- * We also do not need to shutdown workers when we are in enqueue-only mode or we are a
+ /* shut down all workers
+ * We do not need to shutdown workers when we are in enqueue-only mode or we are a
* direct queue - because in both cases we have none... ;)
* with a child! -- rgerhards, 2008-01-28
*/
if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL)
- qqueueShutdownWorkers(pThis);
+ ShutdownWorkers(pThis);
+
+ if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) {
+ CHKiRet(DoSaveOnShutdown(pThis));
+ }
/* finally destruct our (regular) worker thread pool
* Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen,
@@ -2050,7 +2151,7 @@ CODESTARTobjDestruct(qqueue)
* if need arises (what I doubt...) -- rgerhards, 2008-01-25
*/
CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) {
- dbgoprint((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet);
+ DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet);
}
/* finally, clean up some simple things... */
@@ -2060,22 +2161,23 @@ CODESTARTobjDestruct(qqueue)
free(pThis->mut);
}
pthread_mutex_destroy(&pThis->mutThrdMgmt);
- pthread_cond_destroy(&pThis->condDAReady);
pthread_cond_destroy(&pThis->notFull);
pthread_cond_destroy(&pThis->notEmpty);
pthread_cond_destroy(&pThis->belowFullDlyWtrMrk);
pthread_cond_destroy(&pThis->belowLightDlyWtrMrk);
DESTROY_ATOMIC_HELPER_MUT(pThis->mutQueueSize);
+ DESTROY_ATOMIC_HELPER_MUT(pThis->mutLogDeq);
/* type-specific destructor */
iRet = pThis->qDestruct(pThis);
- if(pThis->pszFilePrefix != NULL)
- free(pThis->pszFilePrefix);
+ free(pThis->pszFilePrefix);
+ free(pThis->pszSpoolDir);
- if(pThis->pszSpoolDir != NULL)
- free(pThis->pszSpoolDir);
+ /* some queues do not provide stats and thus have no statsobj! */
+ if(pThis->statsobj != NULL)
+ statsobj.Destruct(&pThis->statsobj);
ENDobjDestruct(qqueue)
@@ -2089,13 +2191,13 @@ qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix)
{
DEFiRet;
- if(pThis->pszFilePrefix != NULL)
- free(pThis->pszFilePrefix);
+ free(pThis->pszFilePrefix);
+ pThis->pszFilePrefix = NULL;
if(pszPrefix == NULL) /* just unset the prefix! */
ABORT_FINALIZE(RS_RET_OK);
- if((pThis->pszFilePrefix = malloc(sizeof(uchar) * iLenPrefix + 1)) == NULL)
+ if((pThis->pszFilePrefix = MALLOC(sizeof(uchar) * iLenPrefix + 1)) == NULL)
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
memcpy(pThis->pszFilePrefix, pszPrefix, iLenPrefix + 1);
pThis->lenFilePrefix = iLenPrefix;
@@ -2125,112 +2227,7 @@ finalize_it:
}
-/* enqueue a new user data element
- * Enqueues the new element and awakes worker thread.
- */
-rsRetVal
-qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
-{
- DEFiRet;
- int iCancelStateSave;
- struct timespec t;
-
- ISOBJ_TYPE_assert(pThis, qqueue);
-
- /* first check if we need to discard this message (which will cause CHKiRet() to exit)
- * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize
- * and bRunsDA parameters may not reflect the correct settings here, but they are
- * "good enough" in the sense that they can be used to drive the decision. Valgrind's
- * threading tools may point this access to be an error, but this is done
- * intentional. I do not see this causes problems to us.
- */
- CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr));
-
- /* Please note that this function is not cancel-safe and consequently
- * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE
- * during its execution. If that is not done, race conditions occur if the
- * thread is canceled (most important use case is input module termination).
- * rgerhards, 2008-01-08
- */
- if(pThis->qType != QUEUETYPE_DIRECT) {
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
- d_pthread_mutex_lock(pThis->mut);
- }
-
- /* then check if we need to add an assistance disk queue */
- if(pThis->bIsDA)
- CHKiRet(qqueueChkStrtDA(pThis));
-
- /* handle flow control
- * There are two different flow control mechanisms: basic and advanced flow control.
- * Basic flow control has always been implemented and protects the queue structures
- * in that it makes sure no more data is enqueued than the queue is configured to
- * support. Enhanced flow control is being added today. There are some sources which
- * can easily be stopped, e.g. a file reader. This is the case because it is unlikely
- * that blocking those sources will have negative effects (after all, the file is
- * continued to be written). Other sources can somewhat be blocked (e.g. the kernel
- * log reader or the local log stream reader): in general, nothing is lost if messages
- * from these sources are not picked up immediately. HOWEVER, they can not block for
- * an extended period of time, as this either causes message loss or - even worse - some
- * other bad effects (e.g. unresponsive system in respect to the main system log socket).
- * Finally, there are some (few) sources which can not be blocked at all. UDP syslog is
- * a prime example. If a UDP message is not received, it is simply lost. So we can't
- * do anything against UDP sockets that come in too fast. The core idea of advanced
- * flow control is that we take into account the different natures of the sources and
- * select flow control mechanisms that fit these needs. This also means, in the end
- * result, that non-blockable sources like UDP syslog receive priority in the system.
- * It's a side effect, but a good one ;) -- rgerhards, 2008-03-14
- */
- if(flowCtlType == eFLOWCTL_FULL_DELAY) {
- while(pThis->iQueueSize >= pThis->iFullDlyMrk) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n");
- pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */
- }
- } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) {
- if(pThis->iQueueSize >= pThis->iLightDlyMrk) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n");
- timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */
- pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */
- }
- }
-
- /* from our regular flow control settings, we are now ready to enqueue the object.
- * However, we now need to do a check if the queue permits to add more data. If that
- * is not the case, basic flow control enters the field, which means we wait for
- * the queue to become ready or drop the new message. -- rgerhards, 2008-03-14
- */
- while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize)
- || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0
- && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n");
- timeoutComp(&t, pThis->toEnq);
- if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n");
- objDestruct(pUsr);
- ABORT_FINALIZE(RS_RET_QUEUE_FULL);
- }
- dbgoprint((obj_t*) pThis, "enqueueMsg: wait solved queue full condition, enqueing\n");
- }
-
- /* and finally enqueue the message */
- CHKiRet(qqueueAdd(pThis, pUsr));
- qqueueChkPersist(pThis);
-
-finalize_it:
- if(pThis->qType != QUEUETYPE_DIRECT) {
- /* make sure at least one worker is running. */
- qqueueAdviseMaxWorkers(pThis);
- /* and release the mutex */
- d_pthread_mutex_unlock(pThis->mut);
- pthread_setcancelstate(iCancelStateSave, NULL);
- dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n");
- }
-
- RETiRet;
-}
-
-
-/* enqueue a single data object. This currently is a helper to qqueueMultiEnqObj.
+/* enqueue a single data object.
* Note that the queue mutex MUST already be locked when this function is called.
* rgerhards, 2009-06-16
*/
@@ -2240,14 +2237,11 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
DEFiRet;
struct timespec t;
+ STATSCOUNTER_INC(pThis->ctrEnqueued, pThis->mutCtrEnqueued);
/* first check if we need to discard this message (which will cause CHKiRet() to exit)
*/
- CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr));
+ CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr));
- /* then check if we need to add an assistance disk queue */
- if(pThis->bIsDA)
- CHKiRet(qqueueChkStrtDA(pThis));
-
/* handle flow control
* There are two different flow control mechanisms: basic and advanced flow control.
* Basic flow control has always been implemented and protects the queue structures
@@ -2270,12 +2264,12 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
*/
if(flowCtlType == eFLOWCTL_FULL_DELAY) {
while(pThis->iQueueSize >= pThis->iFullDlyMrk) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n");
+ DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n");
pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */
}
} else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) {
if(pThis->iQueueSize >= pThis->iLightDlyMrk) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n");
+ DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n");
timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */
pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */
}
@@ -2289,10 +2283,12 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize)
|| (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0
&& pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n");
+ DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n");
timeoutComp(&t, pThis->toEnq);
+ STATSCOUNTER_INC(pThis->ctrFull, pThis->mutCtrFull);
+// TODO : handle enqOnly => discard!
if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) {
- dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n");
+ DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n");
objDestruct(pUsr);
ABORT_FINALIZE(RS_RET_QUEUE_FULL);
}
@@ -2301,12 +2297,13 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
/* and finally enqueue the message */
CHKiRet(qqueueAdd(pThis, pUsr));
- qqueueChkPersist(pThis); // TODO: optimize, do in outer function! (but we need parts from v5?)
+ STATSCOUNTER_SETMAX_NOMUT(pThis->ctrMaxqsize, pThis->iQueueSize);
finalize_it:
RETiRet;
}
+/* ------------------------------ multi-enqueue functions ------------------------------ */
/* enqueue multiple user data elements at once. The aim is to provide a faster interface
* for object submission. Uses the multi_submit_t helper object.
* Please note that this function is not cancel-safe and consequently
@@ -2314,9 +2311,12 @@ finalize_it:
* during its execution. If that is not done, race conditions occur if the
* thread is canceled (most important use case is input module termination).
* rgerhards, 2009-06-16
+ * Note: there now exists multiple different functions implementing specially
+ * optimized algorithms for different config cases. -- rgerhards, 2010-06-09
*/
-rsRetVal
-qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub)
+/* now the function for all modes but direct */
+static rsRetVal
+qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub)
{
int iCancelStateSave;
int i;
@@ -2326,79 +2326,91 @@ qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub)
ISOBJ_TYPE_assert(pThis, qqueue);
assert(pMultiSub != NULL);
- if(pThis->qType != QUEUETYPE_DIRECT) {
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
- d_pthread_mutex_lock(pThis->mut);
- }
-
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+ d_pthread_mutex_lock(pThis->mut);
for(i = 0 ; i < pMultiSub->nElem ; ++i) {
localRet = doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i]);
if(localRet != RS_RET_OK && localRet != RS_RET_QUEUE_FULL)
ABORT_FINALIZE(localRet);
}
+ qqueueChkPersist(pThis, pMultiSub->nElem);
finalize_it:
- if(pThis->qType != QUEUETYPE_DIRECT) {
- /* make sure at least one worker is running. */
- qqueueAdviseMaxWorkers(pThis);
- /* and release the mutex */
- d_pthread_mutex_unlock(pThis->mut);
- pthread_setcancelstate(iCancelStateSave, NULL);
- dbgoprint((obj_t*) pThis, "MultiEnqObj advised worker start\n");
+ /* make sure at least one worker is running. */
+ qqueueAdviseMaxWorkers(pThis);
+ /* and release the mutex */
+ d_pthread_mutex_unlock(pThis->mut);
+ pthread_setcancelstate(iCancelStateSave, NULL);
+ DBGOPRINT((obj_t*) pThis, "MultiEnqObj advised worker start\n");
+
+ RETiRet;
+}
+
+/* now, the same function, but for direct mode */
+static rsRetVal
+qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub)
+{
+ int i;
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ assert(pMultiSub != NULL);
+
+ for(i = 0 ; i < pMultiSub->nElem ; ++i) {
+ CHKiRet(qAddDirect(pThis, (void*)pMultiSub->ppMsgs[i]));
}
+finalize_it:
RETiRet;
}
+/* ------------------------------ END multi-enqueue functions ------------------------------ */
-/* set queue mode to enqueue only or not
- * There is one subtle issue: this method may be called during queue
- * construction or while it is running. In the former case, the queue
- * mutex does not yet exist (it is NULL), while in the later case it
- * must be locked. The function detects the state and operates as
- * required.
- * rgerhards, 2008-01-16
+/* enqueue a new user data element in direct mode
+ * NOTE/TODO: This is a TESTER/EXPERIEMENTAL, to be changed to better
+ * code later on (like multi submit!) 2010-06-10
+ * Enqueues the new element and awakes worker thread.
*/
-static rsRetVal
-qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex)
+rsRetVal
+qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr)
{
DEFiRet;
- DEFVARS_mutexProtection;
+ ISOBJ_TYPE_assert(pThis, qqueue);
+ iRet = qAddDirect(pThis, pUsr);
+ RETiRet;
+}
+
+
+/* enqueue a new user data element
+ * Enqueues the new element and awakes worker thread.
+ */
+rsRetVal
+qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
+{
+ DEFiRet;
+ int iCancelStateSave;
ISOBJ_TYPE_assert(pThis, qqueue);
- /* for simplicity, we do one big mutex lock. This method is extremely seldom
- * called, so that doesn't matter... -- rgerhards, 2008-01-16
- */
- if(pThis->mut != NULL) {
- BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex);
- }
-
- if(bEnqOnly == pThis->bEnqOnly)
- FINALIZE; /* no change, nothing to do */
-
- if(pThis->bQueueStarted) {
- /* we need to adjust queue operation only if we are not during initial param setup */
- if(bEnqOnly == 1) {
- /* switch to enqueue-only mode */
- /* this means we need to terminate all workers - that's it... */
- dbgoprint((obj_t*) pThis, "switching to enqueue-only mode, terminating all worker threads\n");
- if(pThis->pWtpReg != NULL)
- wtpWakeupAllWrkr(pThis->pWtpReg);
- if(pThis->pWtpDA != NULL)
- wtpWakeupAllWrkr(pThis->pWtpDA);
- } else {
- /* switch back to regular mode */
- ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); /* we don't need this so far... */
- }
+ if(pThis->qType != QUEUETYPE_DIRECT) {
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+ d_pthread_mutex_lock(pThis->mut);
}
- pThis->bEnqOnly = bEnqOnly;
+ CHKiRet(doEnqSingleObj(pThis, flowCtlType, pUsr));
+
+ qqueueChkPersist(pThis, 1);
finalize_it:
- if(pThis->mut != NULL) {
- END_MTX_PROTECTED_OPERATIONS(pThis->mut);
+ if(pThis->qType != QUEUETYPE_DIRECT) {
+ /* make sure at least one worker is running. */
+ qqueueAdviseMaxWorkers(pThis);
+ /* and release the mutex */
+ d_pthread_mutex_unlock(pThis->mut);
+ pthread_setcancelstate(iCancelStateSave, NULL);
+ DBGOPRINT((obj_t*) pThis, "EnqueueMsg advised worker start\n");
}
+
RETiRet;
}
@@ -2422,6 +2434,7 @@ DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int)
DEFpropSetMeth(qqueue, bSaveOnShutdown, int)
DEFpropSetMeth(qqueue, pUsr, void*)
DEFpropSetMeth(qqueue, iDeqSlowdown, int)
+DEFpropSetMeth(qqueue, iDeqBatchSize, int)
DEFpropSetMeth(qqueue, sizeOnDiskMax, int64)
@@ -2440,8 +2453,6 @@ static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp)
if(isProp("iQueueSize")) {
pThis->iQueueSize = pProp->val.num;
- } else if(isProp("iUngottenObjs")) {
- pThis->iUngottenObjs = pProp->val.num;
} else if(isProp("tVars.disk.sizeOnDisk")) {
pThis->tVars.disk.sizeOnDisk = pProp->val.num;
} else if(isProp("tVars.disk.bytesRead")) {
@@ -2467,6 +2478,9 @@ BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE)
/* request objects we use */
CHKiRet(objUse(glbl, CORE_COMPONENT));
CHKiRet(objUse(strm, CORE_COMPONENT));
+ CHKiRet(objUse(datetime, CORE_COMPONENT));
+ CHKiRet(objUse(errmsg, CORE_COMPONENT));
+ CHKiRet(objUse(statsobj, CORE_COMPONENT));
/* now set our own handlers */
OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty);
diff --git a/runtime/queue.h b/runtime/queue.h
index aafdaa45..97057180 100644
--- a/runtime/queue.h
+++ b/runtime/queue.h
@@ -27,7 +27,18 @@
#include <pthread.h>
#include "obj.h"
#include "wtp.h"
+#include "batch.h"
#include "stream.h"
+#include "statsobj.h"
+
+/* support for the toDelete list */
+typedef struct toDeleteLst_s toDeleteLst_t;
+struct toDeleteLst_s {
+ qDeqID deqID;
+ int nElemDeq; /* numbe of elements that were dequeued and as such must now be discarded */
+ struct toDeleteLst_s *pNext;
+};
+
/* queue types */
typedef enum {
@@ -44,24 +55,15 @@ typedef struct qLinkedList_S {
} qLinkedList_t;
-typedef struct qWrkThrd_s {
- pthread_t thrdID; /* thread ID */
- qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */
- obj_t *pUsr; /* current user object being processed (or NULL if none) */
- struct queue_s *pQueue; /* my queue (important if only the work thread instance is passed! */
- int iThrd; /* my worker thread array index */
- pthread_cond_t condInitDone; /* signaled when the thread startup is done (once per thread existance) */
- pthread_mutex_t mut;
-} qWrkThrd_t; /* type for queue worker threads */
-
/* the queue object */
-typedef struct queue_s {
+struct queue_s {
BEGINobjInstance;
queueType_t qType;
- bool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */
- bool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */
- bool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */
- bool bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */
+ int nLogDeq; /* number of elements currently logically dequeued */
+ int bShutdownImmediate; /* should all workers cease processing messages? */
+ sbool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */
+ sbool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */
+ sbool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */
int iQueueSize; /* Current number of elements in the queue */
int iMaxQueueSize; /* how large can the queue grow? */
int iNumWorkerThreads;/* number of worker threads to use */
@@ -72,18 +74,20 @@ typedef struct queue_s {
void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */
int iUpdsSincePersist;/* nbr of queue updates since the last persist call */
int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */
- bool bSyncQueueFiles;/* if working with files, sync them after each write? */
+ sbool bSyncQueueFiles;/* if working with files, sync them after each write? */
int iHighWtrMrk; /* high water mark for disk-assisted memory queues */
int iLowWtrMrk; /* low water mark for disk-assisted memory queues */
int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */
int iFullDlyMrk; /* if the queue is above this mark, FULL_DELAYable message are put on hold */
int iLightDlyMrk; /* if the queue is above this mark, LIGHT_DELAYable message are put on hold */
int iDiscardSeverity;/* messages of this severity above are discarded on too-full queue */
- bool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */
+ sbool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */
int toQShutdown; /* timeout for regular queue shutdown in ms */
int toActShutdown; /* timeout for long-running action shutdown in ms */
int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */
+ toDeleteLst_t *toDeleteLst;/* this queue's to-delete list */
int toEnq; /* enqueue timeout */
+ int iDeqBatchSize; /* max number of elements that shall be dequeued at once */
/* rate limiting settings (will be expanded) */
int iDeqSlowdown; /* slow down dequeue by specified nbr of microseconds */
/* end rate limiting */
@@ -97,27 +101,29 @@ typedef struct queue_s {
* applied to detect user configuration errors (and tell me how should we detect what
* the user really wanted...). -- rgerhards, 2008-04-02
*/
- /* ane dequeue time window */
- rsRetVal (*pConsumer)(void *,void*); /* user-supplied consumer function for dequeued messages */
+ /* end dequeue time window */
+ rsRetVal (*pConsumer)(void *,batch_t*,int*); /* user-supplied consumer function for dequeued messages */
/* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the
- * user pointer that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 is pointer
- * to message)
- * rgerhards, 2008-01-28
+ * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2
+ * is pointer to an array of message message pointers), arg3 is a pointer to an interger which is zero
+ * during normal operations and one if the consumer must urgently shut down.
*/
/* type-specific handlers (set during construction) */
rsRetVal (*qConstruct)(struct queue_s *pThis);
rsRetVal (*qDestruct)(struct queue_s *pThis);
rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr);
- rsRetVal (*qDel)(struct queue_s *pThis, void **ppUsr);
+ rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr);
+ rsRetVal (*qDel)(struct queue_s *pThis);
/* end type-specific handler */
+ /* public entry points (set during construction, permit to set best algorithm for params selected) */
+ rsRetVal (*MultiEnq)(qqueue_t *pThis, multi_submit_t *pMultiSub);
+ /* end public entry points */
/* synchronization variables */
pthread_mutex_t mutThrdMgmt; /* mutex for the queue's thread management */
pthread_mutex_t *mut; /* mutex for enqueing and dequeueing messages */
pthread_cond_t notFull, notEmpty;
pthread_cond_t belowFullDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */
pthread_cond_t belowLightDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */
- pthread_cond_t condDAReady;/* signalled when the DA queue is fully initialized and ready for processing */
- int bChildIsDone; /* set to 1 when the child DA queue has finished processing, 0 otherwise */
int bThrdStateChanged; /* at least one thread state has changed if 1 */
/* end sync variables */
/* the following variables are always present, because they
@@ -132,43 +138,40 @@ typedef struct queue_s {
int iNumberFiles; /* how many files make up the queue? */
int64 iMaxFileSize; /* max size for a single queue file */
int64 sizeOnDiskMax; /* maximum size on disk allowed */
+ qDeqID deqIDAdd; /* next dequeue ID to use during add to queue store */
+ qDeqID deqIDDel; /* queue store delete position */
int bIsDA; /* is this queue disk assisted? */
- int bRunsDA; /* is this queue actually *running* disk assisted? */
struct queue_s *pqDA; /* queue for disk-assisted modes */
struct queue_s *pqParent;/* pointer to the parent (if this is a child queue) */
int bDAEnqOnly; /* EnqOnly setting for DA queue */
- /* some data elements for the queueUngetObj() functionality. This list should always be short
- * and is always kept in memory
- */
- qLinkedList_t *pUngetRoot;
- qLinkedList_t *pUngetLast;
- int iUngottenObjs; /* number of objects currently in the "ungotten" list */
/* now follow queueing mode specific data elements */
union { /* different data elements based on queue type (qType) */
struct {
- long head, tail;
+ long deqhead, head, tail;
void** pBuf; /* the queued user data structure */
} farray;
struct {
- qLinkedList_t *pRoot;
+ qLinkedList_t *pDeqRoot;
+ qLinkedList_t *pDelRoot;
qLinkedList_t *pLast;
} linklist;
struct {
int64 sizeOnDisk; /* current amount of disk space used */
int64 bytesRead; /* number of bytes read from current (undeleted!) file */
- strm_t *pWrite; /* current file to be written */
- strm_t *pRead; /* current file to be read */
+ strm_t *pWrite; /* current file to be written */
+ strm_t *pReadDeq; /* current file for dequeueing */
+ strm_t *pReadDel; /* current file for deleting */
} disk;
} tVars;
DEF_ATOMIC_HELPER_MUT(mutQueueSize);
-} qqueue_t;
-
-/* some symbolic constants for easier reference */
-#define QUEUE_MODE_ENQDEQ 0
-#define QUEUE_MODE_ENQONLY 1
+ DEF_ATOMIC_HELPER_MUT(mutLogDeq);
+ /* for statistics subsystem */
+ statsobj_t *statsobj;
+ STATSCOUNTER_DEF(ctrEnqueued, mutCtrEnqueued);
+ STATSCOUNTER_DEF(ctrFull, mutCtrFull);
+ int ctrMaxqsize;
+};
-#define QUEUE_IDX_DA_WORKER 0 /* index for the DA worker (fixed) */
-#define QUEUE_PTR_DA_WORKER(x) (&((pThis)->pWrkThrds[0]))
/* the define below is an "eternal" timeout for the timeout settings which require a value.
* It is one day, which is not really eternal, but comes close to it if we think about
@@ -179,13 +182,14 @@ typedef struct queue_s {
/* prototypes */
rsRetVal qqueueDestruct(qqueue_t **ppThis);
-rsRetVal qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub);
+rsRetVal qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr);
rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr);
rsRetVal qqueueStart(qqueue_t *pThis);
rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize);
rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix);
rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads,
- int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*));
+ int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*));
+rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch);
PROTOTYPEObjClassInit(qqueue);
PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int);
PROTOTYPEpropSetMeth(qqueue, bSyncQueueFiles, int);
@@ -204,6 +208,7 @@ PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int);
PROTOTYPEpropSetMeth(qqueue, pUsr, void*);
PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int);
PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64);
+PROTOTYPEpropSetMeth(qqueue, iDeqBatchSize, int);
#define qqueueGetID(pThis) ((unsigned long) pThis)
#endif /* #ifndef QUEUE_H_INCLUDED */
diff --git a/runtime/regexp.c b/runtime/regexp.c
index 86b3e6c4..21079f80 100644
--- a/runtime/regexp.c
+++ b/runtime/regexp.c
@@ -35,6 +35,7 @@
#include "regexp.h"
MODULE_TYPE_LIB
+MODULE_TYPE_NOKEEP
/* static data */
DEFobjStaticHelpers
diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c
index c209ae30..bdb1c9ff 100644
--- a/runtime/rsyslog.c
+++ b/runtime/rsyslog.c
@@ -80,8 +80,17 @@
#include "prop.h"
#include "rule.h"
#include "ruleset.h"
+#include "parser.h"
+#include "strgen.h"
+#include "statsobj.h"
#include "atomic.h"
+#ifdef HAVE_PTHREAD_SETSCHEDPARAM
+struct sched_param default_sched_param;
+pthread_attr_t default_thread_attr;
+int default_thr_sched_policy;
+#endif
+
/* forward definitions */
static rsRetVal dfltErrLogger(int, uchar *errMsg);
@@ -136,6 +145,18 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF)
if(iRefCount == 0) {
/* init runtime only if not yet done */
+#ifdef HAVE_PTHREAD_SETSCHEDPARAM
+ CHKiRet(pthread_getschedparam(pthread_self(),
+ &default_thr_sched_policy,
+ &default_sched_param));
+ CHKiRet(pthread_attr_init(&default_thread_attr));
+ CHKiRet(pthread_attr_setschedpolicy(&default_thread_attr,
+ default_thr_sched_policy));
+ CHKiRet(pthread_attr_setschedparam(&default_thread_attr,
+ &default_sched_param));
+ CHKiRet(pthread_attr_setinheritsched(&default_thread_attr,
+ PTHREAD_EXPLICIT_SCHED));
+#endif
if(ppErrObj != NULL) *ppErrObj = "obj";
CHKiRet(objClassInit(NULL)); /* *THIS* *MUST* always be the first class initilizer being called! */
CHKiRet(objGetObjInterface(pObjIF)); /* this provides the root pointer for all other queries */
@@ -148,12 +169,12 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF)
* class immediately after it is initialized. And, of course, we load those classes
* first that we use ourselfs... -- rgerhards, 2008-03-07
*/
+ if(ppErrObj != NULL) *ppErrObj = "statsobj";
+ CHKiRet(statsobjClassInit(NULL));
if(ppErrObj != NULL) *ppErrObj = "prop";
CHKiRet(propClassInit(NULL));
if(ppErrObj != NULL) *ppErrObj = "glbl";
CHKiRet(glblClassInit(NULL));
- if(ppErrObj != NULL) *ppErrObj = "datetime";
- CHKiRet(datetimeClassInit(NULL));
if(ppErrObj != NULL) *ppErrObj = "msg";
CHKiRet(msgClassInit(NULL));
if(ppErrObj != NULL) *ppErrObj = "ctok_token";
@@ -184,6 +205,10 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF)
CHKiRet(qqueueClassInit(NULL));
if(ppErrObj != NULL) *ppErrObj = "conf";
CHKiRet(confClassInit(NULL));
+ if(ppErrObj != NULL) *ppErrObj = "parser";
+ CHKiRet(parserClassInit(NULL));
+ if(ppErrObj != NULL) *ppErrObj = "strgen";
+ CHKiRet(strgenClassInit(NULL));
/* dummy "classes" */
if(ppErrObj != NULL) *ppErrObj = "str";
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 03f5120d..69b3c8d1 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -25,6 +25,14 @@
*/
#ifndef INCLUDED_RSYSLOG_H
#define INCLUDED_RSYSLOG_H
+#include <pthread.h>
+#include "typedefs.h"
+
+/* ############################################################# *
+ * # Some constant values # *
+ * ############################################################# */
+#define CONST_LEN_TIMESTAMP_3164 15 /* number of chars (excluding \0!) in a RFC3164 timestamp */
+#define CONST_LEN_TIMESTAMP_3339 32 /* number of chars (excluding \0!) in a RFC3339 timestamp */
/* ############################################################# *
* # Config Settings # *
@@ -40,7 +48,27 @@
#define CONF_TAG_BUFSIZE 32
#define CONF_HOSTNAME_BUFSIZE 32
#define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */
-
+#define CONF_MIN_SIZE_FOR_COMPRESS 60 /* config param: minimum message size to try compression. The smaller
+ * the message, the less likely is any compression gain. We check for
+ * gain before we submit the message. But to do so we still need to
+ * do the (costly) compress() call. The following setting sets a size
+ * for which no call to compress() is done at all. This may result in
+ * a few more bytes being transmited but better overall performance.
+ * Note: I have not yet checked the minimum UDP packet size. It might be
+ * that we do not save anything by compressing very small messages, because
+ * UDP might need to pad ;)
+ * rgerhards, 2006-11-30
+ */
+
+#define CONF_OMOD_NUMSTRINGS_MAXSIZE 2 /* cache for pointers to output module buffer pointers. All
+ * rsyslog-provided plugins do NOT need more than two buffers. If
+ * more are needed (future developments, third-parties), rsyslog
+ * must be recompiled with a larger parameter. Hardcoding this
+ * saves us some overhead, both in runtime in code complexity. As
+ * it is doubtful if ever more than 2 parameters are needed, the
+ * approach taken here is considered appropriate.
+ * rgerhards, 2010-06-24
+ */
/* ############################################################# *
* # End Config Settings # *
@@ -60,104 +88,13 @@
#endif
-/* define some base data types */
-
-typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */
-typedef struct thrdInfo thrdInfo_t;
-typedef struct obj_s obj_t;
-typedef struct ruleset_s ruleset_t;
-typedef struct rule_s rule_t;
-//typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */
-typedef struct NetAddr netAddr_t;
-typedef struct netstrms_s netstrms_t;
-typedef struct netstrm_s netstrm_t;
-typedef struct nssel_s nssel_t;
-typedef enum nsdsel_waitOp_e nsdsel_waitOp_t;
-typedef struct nsd_ptcp_s nsd_ptcp_t;
-typedef struct nsd_gtls_s nsd_gtls_t;
-typedef struct nsd_gsspi_s nsd_gsspi_t;
-typedef struct nsd_nss_s nsd_nss_t;
-typedef struct nsdsel_ptcp_s nsdsel_ptcp_t;
-typedef struct nsdsel_gtls_s nsdsel_gtls_t;
-typedef struct msg msg_t;
-typedef struct prop_s prop_t;
-typedef struct interface_s interface_t;
-typedef struct objInfo_s objInfo_t;
-typedef enum rsRetVal_ rsRetVal; /**< friendly type for global return value */
-typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function ptr to a function returning a function ptr... */
-typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */
-typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */
-typedef struct tcpsrv_s tcpsrv_t;
-typedef struct tcps_sess_s tcps_sess_t;
-typedef struct strmsrv_s strmsrv_t;
-typedef struct strms_sess_s strms_sess_t;
-typedef struct vmstk_s vmstk_t;
-typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */
-
-typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename?
-typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename?
-
-/* under Solaris (actually only SPARC), we need to redefine some types
- * to be void, so that we get void* pointers. Otherwise, we will see
- * alignment errors.
- */
-#ifdef OS_SOLARIS
- typedef void * obj_t_ptr;
- typedef void nsd_t;
- typedef void nsdsel_t;
-#else
- typedef obj_t *obj_t_ptr;
- typedef obj_t nsd_t;
- typedef obj_t nsdsel_t;
-#endif
-
-
-/* some universal 64 bit define... */
-typedef long long int64;
-typedef long long unsigned uint64;
-typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */
-typedef char intTiny; /* 0..127! */
-typedef uchar uintTiny; /* 0..255! */
-
-#ifdef __hpux
-typedef unsigned int u_int32_t; /* TODO: is this correct? */
-typedef int socklen_t;
-#endif
-
-typedef char bool; /* I intentionally use char, to keep it slim so that many fit into the CPU cache! */
-
-/* settings for flow control
- * TODO: is there a better place for them? -- rgerhards, 2008-03-14
- */
-typedef enum {
- eFLOWCTL_NO_DELAY = 0, /**< UDP and other non-delayable sources */
- eFLOWCTL_LIGHT_DELAY = 1, /**< some light delay possible, but no extended period of time */
- eFLOWCTL_FULL_DELAY = 2 /**< delay possible for extended period of time */
-} flowControl_t;
-
-/* filter operations */
-typedef enum {
- FIOP_NOP = 0, /* do not use - No Operation */
- FIOP_CONTAINS = 1, /* contains string? */
- FIOP_ISEQUAL = 2, /* is (exactly) equal? */
- FIOP_STARTSWITH = 3, /* starts with a string? */
- FIOP_REGEX = 4, /* matches a (BRE) regular expression? */
- FIOP_EREREGEX = 5 /* matches a ERE regular expression? */
-} fiop_t;
-
-
-/* multi-submit support.
- * This is done via a simple data structure, which holds the number of elements
- * as well as an array of to-be-submitted messages.
- * rgerhards, 2009-06-16
+/* the rsyslog core provides information about present feature to plugins
+ * asking it. Below are feature-test macros which must be used to query
+ * features. Note that this must be powers of two, so that multiple queries
+ * can be combined. -- rgerhards, 2009-04-27
*/
-typedef struct multi_submit_s multi_submit_t;
-struct multi_submit_s {
- short maxElem; /* maximum number of Elements */
- short nElem; /* current number of Elements, points to the next one FREE */
- msg_t **ppMsgs;
-};
-
+#define CORE_FEATURE_BATCHING 1
+/*#define CORE_FEATURE_whatever 2 ... and so on ... */
#ifndef _PATH_CONSOLE
#define _PATH_CONSOLE "/dev/console"
@@ -200,6 +137,7 @@ typedef uintTiny propid_t;
#define PROP_SYS_QHOUR 156
#define PROP_SYS_MINUTE 157
#define PROP_SYS_MYHOSTNAME 158
+#define PROP_SYS_BOM 159
/* The error codes below are orginally "borrowed" from
@@ -364,7 +302,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth
RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */
RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */
RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */
- RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */
+ RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended */
RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */
RS_RET_NON_SIZELIMITCMD = -2125, /**< size limit for file defined, but no size limit command given */
RS_RET_SIZELIMITCMD_DIDNT_RESOLVE = -2126, /**< size limit command did not resolve situation */
@@ -377,21 +315,47 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth
RS_RET_ERR_OPEN_KLOG = -2145, /**< error opening the kernel log socket (primarily solaris) */
RS_RET_ERR_AQ_CONLOG = -2146, /**< error aquiring console log (on solaris) */
RS_RET_ERR_DOOR = -2147, /**< some problems with handling the Solaris door functionality */
+ RS_RET_NO_SRCNAME_TPL = -2150, /**< sourcename template was not specified where one was needed (omudpspoof spoof addr) */
+ RS_RET_HOST_NOT_SPECIFIED = -2151, /**< (target) host was not specified where it was needed */
+ RS_RET_ERR_LIBNET_INIT = -2152, /**< error initializing libnet */
+ RS_RET_FORCE_TERM = -2153, /**< thread was forced to terminate by bShallShutdown, a state, not an error */
+ RS_RET_RULES_QUEUE_EXISTS = -2154,/**< we were instructed to create a new ruleset queue, but one already exists */
+ RS_RET_NO_CURR_RULESET = -2155,/**< no current ruleset exists (but one is required) */
+ RS_RET_NO_MSG_PASSING = -2156,/**< output module interface parameter passing mode "MSG" is not available but required */
+ RS_RET_RULESET_NOT_FOUND = -2157,/**< a required ruleset could not be found */
+ RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */
+ RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */
+ RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */
+ RS_RET_EINTR = -2161, /**< EINTR occured during a system call (not necessarily an error) */
+ RS_RET_ERR_EPOLL = -2162, /**< epoll() returned with an unexpected error code */
+ RS_RET_ERR_EPOLL_CTL = -2163, /**< epol_ctll() returned with an unexpected error code */
+ RS_RET_TIMEOUT = -2164, /**< timeout occured during operation */
+ RS_RET_RCV_ERR = -2165, /**< error occured during socket rcv operation */
RS_RET_NO_SOCK_CONFIGURED = -2166, /**< no socket (name) was configured where one is required */
RS_RET_NO_LSTN_DEFINED = -2172, /**< no listener defined (e.g. inside an input module) */
RS_RET_EPOLL_CR_FAILED = -2173, /**< epoll_create() failed */
RS_RET_EPOLL_CTL_FAILED = -2174, /**< epoll_ctl() failed */
RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */
+ RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */
+ RS_RET_RATE_LIMITED = -2177, /**< some messages discarded due to exceeding a rate limit */
+ RS_RET_ERR_HDFS_WRITE = -2178, /**< error writing to HDFS */
+ RS_RET_ERR_HDFS_OPEN = -2179, /**< error during hdfsOpen (e.g. file does not exist) */
+ RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */
+ RS_RET_ERR_WRKDIR = -2181, /**< problems with the rsyslog working directory */
+ RS_RET_WRN_WRKDIR = -2182, /**< correctable problems with the rsyslog working directory */
RS_RET_OUTDATED_STMT = -2184, /**< some outdated statement/functionality is being used in conf file */
+ RS_RET_MISSING_WHITESPACE = -2185, /**< whitespace is missing in some config construct */
/* RainerScript error messages (range 1000.. 1999) */
RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */
/* some generic error/status codes */
+ RS_RET_OK = 0, /**< operation successful */
RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */
RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */
RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */
- RS_RET_OK = 0 /**< operation successful */
+ RS_RET_IDLE = 4, /**< operation successful, but callee is idle (e.g. because queue is empty) */
+ RS_RET_TERMINATE_WHEN_IDLE = 5 /**< operation successful, function is requested to terminate when idle */
};
/* some helpful macros to work with srRetVals.
@@ -457,6 +421,12 @@ typedef enum rsObjectID rsObjID;
#define RSFREEOBJ(x) {(x)->OID = OIDrsFreed; free(x);}
#endif
+#ifdef HAVE_PTHREAD_SETSCHEDPARAM
+extern struct sched_param default_sched_param;
+extern pthread_attr_t default_thread_attr;
+extern int default_thr_sched_policy;
+#endif
+
/* for the time being, we do our own portability handling here. It
* looks like autotools either does not yet support checks for it, or
diff --git a/runtime/rule.c b/runtime/rule.c
index 4c2c9edb..0776e2dc 100644
--- a/runtime/rule.c
+++ b/runtime/rule.c
@@ -39,8 +39,8 @@
#include "vm.h"
#include "var.h"
#include "srUtils.h"
+#include "batch.h"
#include "unicode-helper.h"
-#include "dirty.h" /* for getFIOPName */
/* static data */
DEFobjStaticHelpers
@@ -49,6 +49,35 @@ DEFobjCurrIf(expr)
DEFobjCurrIf(var)
DEFobjCurrIf(vm)
+
+/* support for simple textual representation of FIOP names
+ * rgerhards, 2005-09-27
+ */
+static char*
+getFIOPName(unsigned iFIOP)
+{
+ char *pRet;
+ switch(iFIOP) {
+ case FIOP_CONTAINS:
+ pRet = "contains";
+ break;
+ case FIOP_ISEQUAL:
+ pRet = "isequal";
+ break;
+ case FIOP_STARTSWITH:
+ pRet = "startswith";
+ break;
+ case FIOP_REGEX:
+ pRet = "regex";
+ break;
+ default:
+ pRet = "NOP";
+ break;
+ }
+ return pRet;
+}
+
+
/* iterate over all actions, this is often needed, for example when HUP processing
* must be done or a shutdown is pending.
*/
@@ -59,40 +88,20 @@ iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam)
}
-
/* helper to processMsg(), used to call the configured actions. It is
* executed from within llExecFunc() of the action list.
* rgerhards, 2007-08-02
*/
-typedef struct processMsgDoActions_s {
- int bPrevWasSuspended; /* was the previous action suspended? */
- msg_t *pMsg;
-} processMsgDoActions_t;
-DEFFUNC_llExecFunc(processMsgDoActions)
+DEFFUNC_llExecFunc(processBatchDoActions)
{
DEFiRet;
rsRetVal iRetMod; /* return value of module - we do not always pass that back */
action_t *pAction = (action_t*) pData;
- processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam;
-
- assert(pAction != NULL);
+ batch_t *pBatch = (batch_t*) pParam;
- if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) {
- dbgprintf("not calling action because the previous one is not suspended\n");
- ABORT_FINALIZE(RS_RET_OK);
- }
-
- iRetMod = actionCallAction(pAction, pDoActData->pMsg);
- if(iRetMod == RS_RET_DISCARDMSG) {
- ABORT_FINALIZE(RS_RET_DISCARDMSG);
- } else if(iRetMod == RS_RET_SUSPENDED) {
- /* indicate suspension for next module to be called */
- pDoActData->bPrevWasSuspended = 1;
- } else {
- pDoActData->bPrevWasSuspended = 0;
- }
+ DBGPRINTF("Processing next action\n");
+ iRetMod = pAction->submitToActQ(pAction, pBatch);
-finalize_it:
RETiRet;
}
@@ -101,7 +110,7 @@ finalize_it:
* provided filter condition.
*/
static rsRetVal
-shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg)
+shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, sbool *bProcessMsg)
{
DEFiRet;
unsigned short pbMustBeFreed;
@@ -164,7 +173,7 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg)
if(pRule->f_filter_type == FILTER_PRI) {
/* skip messages that are incorrect priority */
-dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility]);
+ dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility]);
if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \
((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) )
bRet = 0;
@@ -176,7 +185,7 @@ dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFac
CHKiRet(vm.SetMsg(pVM, pMsg));
CHKiRet(vm.ExecProg(pVM, pRule->f_filterData.f_expr->pVmprg));
CHKiRet(vm.PopBoolFromStack(pVM, &pResult));
- dbgprintf("result of expression evaluation: %lld\n", pResult->val.num);
+ dbgprintf("result of rainerscript filter evaluation: %lld\n", pResult->val.num);
/* VM is destructed on function exit */
bRet = (pResult->val.num) ? 1 : 0;
} else {
@@ -250,26 +259,34 @@ finalize_it:
-/* Process (consume) a received message. Calls the actions configured.
+/* Process (consume) a batch of messages. Calls the actions configured.
* rgerhards, 2005-10-13
*/
static rsRetVal
-processMsg(rule_t *pThis, msg_t *pMsg)
+processBatch(rule_t *pThis, batch_t *pBatch)
{
- int bProcessMsg;
- processMsgDoActions_t DoActData;
+ int i;
+ rsRetVal localRet;
DEFiRet;
ISOBJ_TYPE_assert(pThis, rule);
- assert(pMsg != NULL);
-
- /* first check the filters... */
- CHKiRet(shouldProcessThisMessage(pThis, pMsg, &bProcessMsg));
- if(bProcessMsg) {
- DoActData.pMsg = pMsg;
- DoActData.bPrevWasSuspended = 0;
- CHKiRet(llExecFunc(&pThis->llActList, processMsgDoActions, (void*)&DoActData));
+ assert(pBatch != NULL);
+
+ /* first check the filters and reset status variables */
+ for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+ localRet = shouldProcessThisMessage(pThis, (msg_t*)(pBatch->pElem[i].pUsrp),
+ &(pBatch->pElem[i].bFilterOK));
+ if(localRet != RS_RET_OK) {
+ DBGPRINTF("processBatch: iRet %d returned from shouldProcessThisMessage, "
+ "ignoring message\n", localRet);
+ pBatch->pElem[i].bFilterOK = 0;
+ }
+ if(pBatch->pElem[i].bFilterOK) {
+ /* re-init only when actually needed (cache write cost!) */
+ pBatch->pElem[i].bPrevWasSuspended = 0;
+ }
}
+ CHKiRet(llExecFunc(&pThis->llActList, processBatchDoActions, pBatch));
finalize_it:
RETiRet;
@@ -412,7 +429,7 @@ CODESTARTobjQueryInterface(rule)
pIf->DebugPrint = ruleDebugPrint;
pIf->IterateAllActions = iterateAllActions;
- pIf->ProcessMsg = processMsg;
+ pIf->ProcessBatch = processBatch;
pIf->SetAssRuleset = setAssRuleset;
pIf->GetAssRuleset = getAssRuleset;
finalize_it:
diff --git a/runtime/rule.h b/runtime/rule.h
index 99ac44e7..309a2ed8 100644
--- a/runtime/rule.h
+++ b/runtime/rule.h
@@ -47,7 +47,7 @@ struct rule_s {
fiop_t operation;
regex_t *regex_cache; /* cache for compiled REs, if such are used */
cstr_t *pCSCompValue; /* value to "compare" against */
- bool isNegated;
+ sbool isNegated;
propid_t propID; /* ID of the requested property */
} prop;
expr_t *f_expr; /* expression object */
@@ -64,11 +64,12 @@ BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */
rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis);
rsRetVal (*Destruct)(rule_t **ppThis);
rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam);
- rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg);
+ rsRetVal (*ProcessBatch)(rule_t *pThis, batch_t *pBatch);
rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*);
ruleset_t* (*GetAssRuleset)(rule_t *pThis);
ENDinterface(rule)
-#define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
+#define ruleCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */
+/* change for v2: ProcessMsg replaced by ProcessBatch - 2010-06-10 */
/* prototypes */
diff --git a/runtime/ruleset.c b/runtime/ruleset.c
index af61f24f..5ee2a55a 100644
--- a/runtime/ruleset.c
+++ b/runtime/ruleset.c
@@ -40,22 +40,28 @@
#include "rsyslog.h"
#include "obj.h"
+#include "cfsysline.h"
#include "msg.h"
#include "ruleset.h"
#include "rule.h"
#include "errmsg.h"
+#include "parser.h"
+#include "batch.h"
#include "unicode-helper.h"
-
-static rsRetVal debugPrintAll(void); // TODO: remove!
+#include "dirty.h" /* for main ruleset queue creation */
/* static data */
DEFobjStaticHelpers
DEFobjCurrIf(errmsg)
DEFobjCurrIf(rule)
+DEFobjCurrIf(parser)
linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */
ruleset_t *pCurrRuleset = NULL; /* currently "active" ruleset */
-ruleset_t *pDfltRuleset = NULL; /* currentl default ruleset, e.g. for binding to actions which have no other */
+ruleset_t *pDfltRuleset = NULL; /* current default ruleset, e.g. for binding to actions which have no other */
+
+/* forward definitions */
+static rsRetVal processBatch(batch_t *pBatch);
/* ---------- linked-list key handling functions ---------- */
@@ -132,39 +138,119 @@ finalize_it:
-/* helper to processMsg(), used to call the configured actions. It is
+/* helper to processBatch(), used to call the configured actions. It is
* executed from within llExecFunc() of the action list.
* rgerhards, 2007-08-02
*/
-DEFFUNC_llExecFunc(processMsgDoRules)
+DEFFUNC_llExecFunc(processBatchDoRules)
{
+ rsRetVal iRet;
ISOBJ_TYPE_assert(pData, rule);
- return rule.ProcessMsg((rule_t*) pData, (msg_t*) pParam);
+ dbgprintf("Processing next rule\n");
+ iRet = rule.ProcessBatch((rule_t*) pData, (batch_t*) pParam);
+dbgprintf("ruleset: get iRet %d from rule.ProcessMsg()\n", iRet);
+ return iRet;
}
-/* Process (consume) a received message. Calls the actions configured.
+
+/* This function is similar to processBatch(), but works on a batch that
+ * contains rules from multiple rulesets. In this case, we can not push
+ * the whole batch through the ruleset. Instead, we examine it and
+ * partition it into sub-rulesets which we then push through the system.
+ * Note that when we evaluate which message must be processed, we do NOT need
+ * to look at bFilterOK, because this value is only set in a later processing
+ * stage. Doing so caused a bug during development ;)
+ * rgerhards, 2010-06-15
+ */
+static inline rsRetVal
+processBatchMultiRuleset(batch_t *pBatch)
+{
+ ruleset_t *currRuleset;
+ batch_t snglRuleBatch;
+ int i;
+ int iStart; /* start index of partial batch */
+ int iNew; /* index for new (temporary) batch */
+ int bHaveUnprocessed; /* do we (still) have unprocessed entries? (loop term predicate) */
+ DEFiRet;
+
+ do {
+ bHaveUnprocessed = 0;
+ /* search for first unprocessed element */
+ for(iStart = 0 ; iStart < pBatch->nElem && pBatch->pElem[iStart].state == BATCH_STATE_DISC ; ++iStart)
+ /* just search, no action */;
+ if(iStart == pBatch->nElem)
+ break; /* everything processed */
+
+ /* prepare temporary batch */
+ CHKiRet(batchInit(&snglRuleBatch, pBatch->nElem));
+ snglRuleBatch.pbShutdownImmediate = pBatch->pbShutdownImmediate;
+ currRuleset = batchElemGetRuleset(pBatch, iStart);
+ iNew = 0;
+ for(i = iStart ; i < pBatch->nElem ; ++i) {
+ if(batchElemGetRuleset(pBatch, i) == currRuleset) {
+ /* for performance reasons, we copy only those members that we actually need */
+ snglRuleBatch.pElem[iNew].pUsrp = pBatch->pElem[i].pUsrp;
+ snglRuleBatch.pElem[iNew].state = pBatch->pElem[i].state;
+ ++iNew;
+ /* We indicate the element also as done, so it will not be processed again */
+ pBatch->pElem[i].state = BATCH_STATE_DISC;
+ } else {
+ bHaveUnprocessed = 1;
+ }
+ }
+ snglRuleBatch.nElem = iNew; /* was left just right by the for loop */
+ batchSetSingleRuleset(&snglRuleBatch, 1);
+ /* process temp batch */
+ processBatch(&snglRuleBatch);
+ batchFree(&snglRuleBatch);
+ } while(bHaveUnprocessed == 1);
+
+finalize_it:
+ RETiRet;
+}
+
+/* Process (consume) a batch of messages. Calls the actions configured.
+ * If the whole batch uses a singel ruleset, we can process the batch as
+ * a whole. Otherwise, we need to process it slower, on a message-by-message
+ * basis (what can be optimized to a per-ruleset basis)
* rgerhards, 2005-10-13
*/
static rsRetVal
-processMsg(msg_t *pMsg)
+processBatch(batch_t *pBatch)
{
ruleset_t *pThis;
DEFiRet;
- assert(pMsg != NULL);
-
- pThis = (pMsg->pRuleset == NULL) ? pDfltRuleset : pMsg->pRuleset;
- ISOBJ_TYPE_assert(pThis, ruleset);
-
- CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg));
+ assert(pBatch != NULL);
+
+ DBGPRINTF("processBatch: batch of %d elements must be processed\n", pBatch->nElem);
+ if(pBatch->bSingleRuleset) {
+ pThis = batchGetRuleset(pBatch);
+ if(pThis == NULL)
+ pThis = pDfltRuleset;
+ ISOBJ_TYPE_assert(pThis, ruleset);
+ CHKiRet(llExecFunc(&pThis->llRules, processBatchDoRules, pBatch));
+ } else {
+ CHKiRet(processBatchMultiRuleset(pBatch));
+ }
finalize_it:
- if(iRet == RS_RET_DISCARDMSG)
- iRet = RS_RET_OK;
-
+ DBGPRINTF("ruleset.ProcessMsg() returns %d\n", iRet);
RETiRet;
}
+
+/* return the ruleset-assigned parser list. NULL means use the default
+ * parser list.
+ * rgerhards, 2009-11-04
+ */
+static parserList_t*
+GetParserList(msg_t *pMsg)
+{
+ return (pMsg->pRuleset == NULL) ? pDfltRuleset->pParserLst : pMsg->pRuleset->pParserLst;
+}
+
+
/* Add a new rule to the end of the current rule set. We do a number
* of checks and ignore the rule if it does not pass them.
*/
@@ -214,6 +300,19 @@ GetCurrent(void)
}
+/* get main queue associated with ruleset. If no ruleset-specifc main queue
+ * is set, the primary main message queue is returned.
+ * We use a non-standard calling interface, as nothing can go wrong and it
+ * is really much more natural to return the pointer directly.
+ */
+static qqueue_t*
+GetRulesetQueue(ruleset_t *pThis)
+{
+ ISOBJ_TYPE_assert(pThis, ruleset);
+ return (pThis->pQueue == NULL) ? pMsgQueue : pThis->pQueue;
+}
+
+
/* Find the ruleset with the given name and return a pointer to its object.
*/
static rsRetVal
@@ -319,6 +418,12 @@ finalize_it:
BEGINobjDestruct(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */
CODESTARTobjDestruct(ruleset)
dbgprintf("destructing ruleset %p, name %p\n", pThis, pThis->pszName);
+ if(pThis->pQueue != NULL) {
+ qqueueDestruct(&pThis->pQueue);
+ }
+ if(pThis->pParserLst != NULL) {
+ parser.DestructParserList(&pThis->pParserLst);
+ }
llDestroy(&pThis->llRules);
free(pThis->pszName);
ENDobjDestruct(ruleset)
@@ -385,6 +490,81 @@ debugPrintAll(void)
}
+/* Create a ruleset-specific "main" queue for this ruleset. If one is already
+ * defined, an error message is emitted but nothing else is done.
+ * Note: we use the main message queue parameters for queue creation and access
+ * syslogd.c directly to obtain these. This is far from being perfect, but
+ * considered acceptable for the time being.
+ * rgerhards, 2009-10-27
+ */
+static rsRetVal
+rulesetCreateQueue(void __attribute__((unused)) *pVal, int *pNewVal)
+{
+ DEFiRet;
+
+ if(pCurrRuleset == NULL) {
+ errmsg.LogError(0, RS_RET_NO_CURR_RULESET, "error: currently no specific ruleset specified, thus a "
+ "queue can not be added to it");
+ ABORT_FINALIZE(RS_RET_NO_CURR_RULESET);
+ }
+
+ if(pCurrRuleset->pQueue != NULL) {
+ errmsg.LogError(0, RS_RET_RULES_QUEUE_EXISTS, "error: ruleset already has a main queue, can not "
+ "add another one");
+ ABORT_FINALIZE(RS_RET_RULES_QUEUE_EXISTS);
+ }
+
+ if(pNewVal == 0)
+ FINALIZE; /* if it is turned off, we do not need to change anything ;) */
+
+ dbgprintf("adding a ruleset-specific \"main\" queue");
+ CHKiRet(createMainQueue(&pCurrRuleset->pQueue, UCHAR_CONSTANT("ruleset")));
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* Add a ruleset specific parser to the ruleset. Note that adding the first
+ * parser automatically disables the default parsers. If they are needed as well,
+ * the must be added via explicit config directives.
+ * Note: this is the only spot in the code that requires the parser object. In order
+ * to solve some class init bootstrap sequence problems, we get the object handle here
+ * instead of during module initialization. Note that objUse() is capable of being
+ * called multiple times.
+ * rgerhards, 2009-11-04
+ */
+static rsRetVal
+rulesetAddParser(void __attribute__((unused)) *pVal, uchar *pName)
+{
+ parser_t *pParser;
+ DEFiRet;
+
+ assert(pCurrRuleset != NULL);
+
+ CHKiRet(objUse(parser, CORE_COMPONENT));
+ iRet = parser.FindParser(&pParser, pName);
+ if(iRet == RS_RET_PARSER_NOT_FOUND) {
+ errmsg.LogError(0, RS_RET_PARSER_NOT_FOUND, "error: parser '%s' unknown at this time "
+ "(maybe defined too late in rsyslog.conf?)", pName);
+ ABORT_FINALIZE(RS_RET_NO_CURR_RULESET);
+ } else if(iRet != RS_RET_OK) {
+ errmsg.LogError(0, iRet, "error trying to find parser '%s'\n", pName);
+ FINALIZE;
+ }
+
+ CHKiRet(parser.AddParserToList(&pCurrRuleset->pParserLst, pParser));
+
+ dbgprintf("added parser '%s' to ruleset '%s'\n", pName, pCurrRuleset->pszName);
+RUNLOG_VAR("%p", pCurrRuleset->pParserLst);
+
+finalize_it:
+ d_free(pName); /* no longer needed */
+
+ RETiRet;
+}
+
+
/* queryInterface function
* rgerhards, 2008-02-21
*/
@@ -407,13 +587,15 @@ CODESTARTobjQueryInterface(ruleset)
pIf->IterateAllActions = iterateAllActions;
pIf->DestructAllActions = destructAllActions;
pIf->AddRule = addRule;
- pIf->ProcessMsg = processMsg;
+ pIf->ProcessBatch = processBatch;
pIf->SetName = setName;
pIf->DebugPrintAll = debugPrintAll;
pIf->GetCurrent = GetCurrent;
pIf->GetRuleset = GetRuleset;
pIf->SetDefaultRuleset = SetDefaultRuleset;
pIf->SetCurrRuleset = SetCurrRuleset;
+ pIf->GetRulesetQueue = GetRulesetQueue;
+ pIf->GetParserList = GetParserList;
finalize_it:
ENDobjQueryInterface(ruleset)
@@ -425,6 +607,7 @@ BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */
llDestroy(&llRulesets);
objRelease(errmsg, CORE_COMPONENT);
objRelease(rule, CORE_COMPONENT);
+ objRelease(parser, CORE_COMPONENT);
ENDObjClassExit(ruleset)
@@ -443,6 +626,10 @@ BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */
/* prepare global data */
CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp));
+
+ /* config file handlers */
+ CHKiRet(regCfSysLineHdlr((uchar *)"rulesetparser", 0, eCmdHdlrGetWord, rulesetAddParser, NULL, NULL));
+ CHKiRet(regCfSysLineHdlr((uchar *)"rulesetcreatemainqueue", 0, eCmdHdlrBinary, rulesetCreateQueue, NULL, NULL));
ENDObjClassInit(ruleset)
/* vi:set ai:
diff --git a/runtime/ruleset.h b/runtime/ruleset.h
index 32571687..acebd17a 100644
--- a/runtime/ruleset.h
+++ b/runtime/ruleset.h
@@ -25,6 +25,7 @@
#ifndef INCLUDED_RULESET_H
#define INCLUDED_RULESET_H
+#include "queue.h"
#include "linkedlist.h"
/* the ruleset object */
@@ -32,6 +33,8 @@ struct ruleset_s {
BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */
linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */
uchar *pszName; /* name of our ruleset */
+ qqueue_t *pQueue; /* "main" message queue, if the ruleset has its own (else NULL) */
+ parserList_t *pParserLst;/* list of parsers to use for this ruleset */
};
/* interfaces */
@@ -45,13 +48,16 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */
rsRetVal (*DestructAllActions)(void);
rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule);
rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName);
- rsRetVal (*ProcessMsg)(msg_t *pMsg);
+ rsRetVal (*ProcessBatch)(batch_t*);
rsRetVal (*GetRuleset)(ruleset_t **ppThis, uchar*);
rsRetVal (*SetDefaultRuleset)(uchar*);
rsRetVal (*SetCurrRuleset)(uchar*);
ruleset_t* (*GetCurrent)(void);
+ qqueue_t* (*GetRulesetQueue)(ruleset_t*);
+ /* v3, 2009-11-04 */
+ parserList_t* (*GetParserList)(msg_t *);
ENDinterface(ruleset)
-#define rulesetCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
+#define rulesetCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */
/* prototypes */
diff --git a/runtime/sd-daemon.c b/runtime/sd-daemon.c
new file mode 100644
index 00000000..9c23b917
--- /dev/null
+++ b/runtime/sd-daemon.c
@@ -0,0 +1,435 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+ Copyright 2010 Lennart Poettering
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation files
+ (the "Software"), to deal in the Software without restriction,
+ including without limitation the rights to use, copy, modify, merge,
+ publish, distribute, sublicense, and/or sell copies of the Software,
+ and to permit persons to whom the Software is furnished to do so,
+ subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+***/
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/fcntl.h>
+#include <netinet/in.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#include "sd-daemon.h"
+
+int sd_listen_fds(int unset_environment) {
+
+#if defined(DISABLE_SYSTEMD) || !defined(__linux__)
+ return 0;
+#else
+ int r, fd;
+ const char *e;
+ char *p = NULL;
+ unsigned long l;
+
+ if (!(e = getenv("LISTEN_PID"))) {
+ r = 0;
+ goto finish;
+ }
+
+ errno = 0;
+ l = strtoul(e, &p, 10);
+
+ if (errno != 0) {
+ r = -errno;
+ goto finish;
+ }
+
+ if (!p || *p || l <= 0) {
+ r = -EINVAL;
+ goto finish;
+ }
+
+ /* Is this for us? */
+ if (getpid() != (pid_t) l) {
+ r = 0;
+ goto finish;
+ }
+
+ if (!(e = getenv("LISTEN_FDS"))) {
+ r = 0;
+ goto finish;
+ }
+
+ errno = 0;
+ l = strtoul(e, &p, 10);
+
+ if (errno != 0) {
+ r = -errno;
+ goto finish;
+ }
+
+ if (!p || *p) {
+ r = -EINVAL;
+ goto finish;
+ }
+
+ for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + (int) l; fd ++) {
+ int flags;
+
+ if ((flags = fcntl(fd, F_GETFD)) < 0) {
+ r = -errno;
+ goto finish;
+ }
+
+ if (flags & FD_CLOEXEC)
+ continue;
+
+ if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) {
+ r = -errno;
+ goto finish;
+ }
+ }
+
+ r = (int) l;
+
+finish:
+ if (unset_environment) {
+ unsetenv("LISTEN_PID");
+ unsetenv("LISTEN_FDS");
+ }
+
+ return r;
+#endif
+}
+
+int sd_is_fifo(int fd, const char *path) {
+ struct stat st_fd;
+
+ if (fd < 0)
+ return -EINVAL;
+
+ memset(&st_fd, 0, sizeof(st_fd));
+ if (fstat(fd, &st_fd) < 0)
+ return -errno;
+
+ if (!S_ISFIFO(st_fd.st_mode))
+ return 0;
+
+ if (path) {
+ struct stat st_path;
+
+ memset(&st_path, 0, sizeof(st_path));
+ if (stat(path, &st_path) < 0) {
+
+ if (errno == ENOENT || errno == ENOTDIR)
+ return 0;
+
+ return -errno;
+ }
+
+ return
+ st_path.st_dev == st_fd.st_dev &&
+ st_path.st_ino == st_fd.st_ino;
+ }
+
+ return 1;
+}
+
+static int sd_is_socket_internal(int fd, int type, int listening) {
+ struct stat st_fd;
+
+ if (fd < 0 || type < 0)
+ return -EINVAL;
+
+ if (fstat(fd, &st_fd) < 0)
+ return -errno;
+
+ if (!S_ISSOCK(st_fd.st_mode))
+ return 0;
+
+ if (type != 0) {
+ int other_type = 0;
+ socklen_t l = sizeof(other_type);
+
+ if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &other_type, &l) < 0)
+ return -errno;
+
+ if (l != sizeof(other_type))
+ return -EINVAL;
+
+ if (other_type != type)
+ return 0;
+ }
+
+ if (listening >= 0) {
+ int accepting = 0;
+ socklen_t l = sizeof(accepting);
+
+ if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &accepting, &l) < 0)
+ return -errno;
+
+ if (l != sizeof(accepting))
+ return -EINVAL;
+
+ if (!accepting != !listening)
+ return 0;
+ }
+
+ return 1;
+}
+
+union sockaddr_union {
+ struct sockaddr sa;
+ struct sockaddr_in in4;
+ struct sockaddr_in6 in6;
+ struct sockaddr_un un;
+ struct sockaddr_storage storage;
+};
+
+int sd_is_socket(int fd, int family, int type, int listening) {
+ int r;
+
+ if (family < 0)
+ return -EINVAL;
+
+ if ((r = sd_is_socket_internal(fd, type, listening)) <= 0)
+ return r;
+
+ if (family > 0) {
+ union sockaddr_union sockaddr;
+ socklen_t l;
+
+ memset(&sockaddr, 0, sizeof(sockaddr));
+ l = sizeof(sockaddr);
+
+ if (getsockname(fd, &sockaddr.sa, &l) < 0)
+ return -errno;
+
+ if (l < sizeof(sa_family_t))
+ return -EINVAL;
+
+ return sockaddr.sa.sa_family == family;
+ }
+
+ return 1;
+}
+
+int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) {
+ union sockaddr_union sockaddr;
+ socklen_t l;
+ int r;
+
+ if (family != 0 && family != AF_INET && family != AF_INET6)
+ return -EINVAL;
+
+ if ((r = sd_is_socket_internal(fd, type, listening)) <= 0)
+ return r;
+
+ memset(&sockaddr, 0, sizeof(sockaddr));
+ l = sizeof(sockaddr);
+
+ if (getsockname(fd, &sockaddr.sa, &l) < 0)
+ return -errno;
+
+ if (l < sizeof(sa_family_t))
+ return -EINVAL;
+
+ if (sockaddr.sa.sa_family != AF_INET &&
+ sockaddr.sa.sa_family != AF_INET6)
+ return 0;
+
+ if (family > 0)
+ if (sockaddr.sa.sa_family != family)
+ return 0;
+
+ if (port > 0) {
+ if (sockaddr.sa.sa_family == AF_INET) {
+ if (l < sizeof(struct sockaddr_in))
+ return -EINVAL;
+
+ return htons(port) == sockaddr.in4.sin_port;
+ } else {
+ if (l < sizeof(struct sockaddr_in6))
+ return -EINVAL;
+
+ return htons(port) == sockaddr.in6.sin6_port;
+ }
+ }
+
+ return 1;
+}
+
+int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) {
+ union sockaddr_union sockaddr;
+ socklen_t l;
+ int r;
+
+ if ((r = sd_is_socket_internal(fd, type, listening)) <= 0)
+ return r;
+
+ memset(&sockaddr, 0, sizeof(sockaddr));
+ l = sizeof(sockaddr);
+
+ if (getsockname(fd, &sockaddr.sa, &l) < 0)
+ return -errno;
+
+ if (l < sizeof(sa_family_t))
+ return -EINVAL;
+
+ if (sockaddr.sa.sa_family != AF_UNIX)
+ return 0;
+
+ if (path) {
+ if (length <= 0)
+ length = strlen(path);
+
+ if (length <= 0)
+ /* Unnamed socket */
+ return l == sizeof(sa_family_t);
+
+ if (path[0])
+ /* Normal path socket */
+ return
+ (l >= sizeof(sa_family_t) + length + 1) &&
+ memcmp(path, sockaddr.un.sun_path, length+1) == 0;
+ else
+ /* Abstract namespace socket */
+ return
+ (l == sizeof(sa_family_t) + length) &&
+ memcmp(path, sockaddr.un.sun_path, length) == 0;
+ }
+
+ return 1;
+}
+
+int sd_notify(int unset_environment, const char *state) {
+#if defined(DISABLE_SYSTEMD) || !defined(__linux__) || !defined(SOCK_CLOEXEC)
+ return 0;
+#else
+ int fd = -1, r;
+ struct msghdr msghdr;
+ struct iovec iovec;
+ union sockaddr_union sockaddr;
+ const char *e;
+
+ if (!state) {
+ r = -EINVAL;
+ goto finish;
+ }
+
+ if (!(e = getenv("NOTIFY_SOCKET")))
+ return 0;
+
+ /* Must be an abstract socket, or an absolute path */
+ if ((e[0] != '@' && e[0] != '/') || e[1] == 0) {
+ r = -EINVAL;
+ goto finish;
+ }
+
+ if ((fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) {
+ r = -errno;
+ goto finish;
+ }
+
+ memset(&sockaddr, 0, sizeof(sockaddr));
+ sockaddr.sa.sa_family = AF_UNIX;
+ strncpy(sockaddr.un.sun_path, e, sizeof(sockaddr.un.sun_path));
+
+ if (sockaddr.un.sun_path[0] == '@')
+ sockaddr.un.sun_path[0] = 0;
+
+ memset(&iovec, 0, sizeof(iovec));
+ iovec.iov_base = (char*) state;
+ iovec.iov_len = strlen(state);
+
+ memset(&msghdr, 0, sizeof(msghdr));
+ msghdr.msg_name = &sockaddr;
+ msghdr.msg_namelen = sizeof(sa_family_t) + strlen(e);
+
+ if (msghdr.msg_namelen > sizeof(struct sockaddr_un))
+ msghdr.msg_namelen = sizeof(struct sockaddr_un);
+
+ msghdr.msg_iov = &iovec;
+ msghdr.msg_iovlen = 1;
+
+ if (sendmsg(fd, &msghdr, MSG_NOSIGNAL) < 0) {
+ r = -errno;
+ goto finish;
+ }
+
+ r = 1;
+
+finish:
+ if (unset_environment)
+ unsetenv("NOTIFY_SOCKET");
+
+ if (fd >= 0)
+ close(fd);
+
+ return r;
+#endif
+}
+
+int sd_notifyf(int unset_environment, const char *format, ...) {
+#if defined(DISABLE_SYSTEMD) || !defined(__linux__)
+ return 0;
+#else
+ va_list ap;
+ char *p = NULL;
+ int r;
+
+ va_start(ap, format);
+ r = vasprintf(&p, format, ap);
+ va_end(ap);
+
+ if (r < 0 || !p)
+ return -ENOMEM;
+
+ r = sd_notify(unset_environment, p);
+ free(p);
+
+ return r;
+#endif
+}
+
+int sd_booted(void) {
+#if defined(DISABLE_SYSTEMD) || !defined(__linux__)
+ return 0;
+#else
+
+ struct stat a, b;
+
+ /* We simply test whether the systemd cgroup hierarchy is
+ * mounted */
+
+ if (lstat("/sys/fs/cgroup", &a) < 0)
+ return 0;
+
+ if (lstat("/sys/fs/cgroup/systemd", &b) < 0)
+ return 0;
+
+ return a.st_dev != b.st_dev;
+#endif
+}
diff --git a/runtime/sd-daemon.h b/runtime/sd-daemon.h
new file mode 100644
index 00000000..45aac8bd
--- /dev/null
+++ b/runtime/sd-daemon.h
@@ -0,0 +1,261 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+#ifndef foosddaemonhfoo
+#define foosddaemonhfoo
+
+/***
+ Copyright 2010 Lennart Poettering
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation files
+ (the "Software"), to deal in the Software without restriction,
+ including without limitation the rights to use, copy, modify, merge,
+ publish, distribute, sublicense, and/or sell copies of the Software,
+ and to permit persons to whom the Software is furnished to do so,
+ subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+***/
+
+#include <sys/types.h>
+#include <inttypes.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ Reference implementation of a few systemd related interfaces for
+ writing daemons. These interfaces are trivial to implement. To
+ simplify porting we provide this reference implementation.
+ Applications are welcome to reimplement the algorithms described
+ here if they do not want to include these two source files.
+
+ The following functionality is provided:
+
+ - Support for logging with log levels on stderr
+ - File descriptor passing for socket-based activation
+ - Daemon startup and status notification
+ - Detection of systemd boots
+
+ You may compile this with -DDISABLE_SYSTEMD to disable systemd
+ support. This makes all those calls NOPs that are directly related to
+ systemd (i.e. only sd_is_xxx() will stay useful).
+
+ Since this is drop-in code we don't want any of our symbols to be
+ exported in any case. Hence we declare hidden visibility for all of
+ them.
+
+ You may find an up-to-date version of these source files online:
+
+ http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h
+ http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c
+
+ This should compile on non-Linux systems, too, but with the
+ exception of the sd_is_xxx() calls all functions will become NOPs.
+
+ See sd-daemon(7) for more information.
+*/
+
+#if (__GNUC__ >= 4)
+#define _sd_printf_attr_(a,b) __attribute__ ((format (printf, a, b)))
+# if defined(SD_EXPORT_SYMBOLS)
+# define _sd_hidden_
+# else
+# define _sd_hidden_ __attribute__ ((visibility("hidden")))
+# endif
+#else
+#define _sd_printf_attr_(a,b)
+#define _sd_hidden_
+#endif
+
+/*
+ Log levels for usage on stderr:
+
+ fprintf(stderr, SD_NOTICE "Hello World!\n");
+
+ This is similar to printk() usage in the kernel.
+*/
+#define SD_EMERG "<0>" /* system is unusable */
+#define SD_ALERT "<1>" /* action must be taken immediately */
+#define SD_CRIT "<2>" /* critical conditions */
+#define SD_ERR "<3>" /* error conditions */
+#define SD_WARNING "<4>" /* warning conditions */
+#define SD_NOTICE "<5>" /* normal but significant condition */
+#define SD_INFO "<6>" /* informational */
+#define SD_DEBUG "<7>" /* debug-level messages */
+
+/* The first passed file descriptor is fd 3 */
+#define SD_LISTEN_FDS_START 3
+
+/*
+ Returns how many file descriptors have been passed, or a negative
+ errno code on failure. Optionally, removes the $LISTEN_FDS and
+ $LISTEN_PID file descriptors from the environment (recommended, but
+ problematic in threaded environments). If r is the return value of
+ this function you'll find the file descriptors passed as fds
+ SD_LISTEN_FDS_START to SD_LISTEN_FDS_START+r-1. Returns a negative
+ errno style error code on failure. This function call ensures that
+ the FD_CLOEXEC flag is set for the passed file descriptors, to make
+ sure they are not passed on to child processes. If FD_CLOEXEC shall
+ not be set, the caller needs to unset it after this call for all file
+ descriptors that are used.
+
+ See sd_listen_fds(3) for more information.
+*/
+int sd_listen_fds(int unset_environment) _sd_hidden_;
+
+/*
+ Helper call for identifying a passed file descriptor. Returns 1 if
+ the file descriptor is a FIFO in the file system stored under the
+ specified path, 0 otherwise. If path is NULL a path name check will
+ not be done and the call only verifies if the file descriptor
+ refers to a FIFO. Returns a negative errno style error code on
+ failure.
+
+ See sd_is_fifo(3) for more information.
+*/
+int sd_is_fifo(int fd, const char *path) _sd_hidden_;
+
+/*
+ Helper call for identifying a passed file descriptor. Returns 1 if
+ the file descriptor is a socket of the specified family (AF_INET,
+ ...) and type (SOCK_DGRAM, SOCK_STREAM, ...), 0 otherwise. If
+ family is 0 a socket family check will not be done. If type is 0 a
+ socket type check will not be done and the call only verifies if
+ the file descriptor refers to a socket. If listening is > 0 it is
+ verified that the socket is in listening mode. (i.e. listen() has
+ been called) If listening is == 0 it is verified that the socket is
+ not in listening mode. If listening is < 0 no listening mode check
+ is done. Returns a negative errno style error code on failure.
+
+ See sd_is_socket(3) for more information.
+*/
+int sd_is_socket(int fd, int family, int type, int listening) _sd_hidden_;
+
+/*
+ Helper call for identifying a passed file descriptor. Returns 1 if
+ the file descriptor is an Internet socket, of the specified family
+ (either AF_INET or AF_INET6) and the specified type (SOCK_DGRAM,
+ SOCK_STREAM, ...), 0 otherwise. If version is 0 a protocol version
+ check is not done. If type is 0 a socket type check will not be
+ done. If port is 0 a socket port check will not be done. The
+ listening flag is used the same way as in sd_is_socket(). Returns a
+ negative errno style error code on failure.
+
+ See sd_is_socket_inet(3) for more information.
+*/
+int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) _sd_hidden_;
+
+/*
+ Helper call for identifying a passed file descriptor. Returns 1 if
+ the file descriptor is an AF_UNIX socket of the specified type
+ (SOCK_DGRAM, SOCK_STREAM, ...) and path, 0 otherwise. If type is 0
+ a socket type check will not be done. If path is NULL a socket path
+ check will not be done. For normal AF_UNIX sockets set length to
+ 0. For abstract namespace sockets set length to the length of the
+ socket name (including the initial 0 byte), and pass the full
+ socket path in path (including the initial 0 byte). The listening
+ flag is used the same way as in sd_is_socket(). Returns a negative
+ errno style error code on failure.
+
+ See sd_is_socket_unix(3) for more information.
+*/
+int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) _sd_hidden_;
+
+/*
+ Informs systemd about changed daemon state. This takes a number of
+ newline separated environment-style variable assignments in a
+ string. The following variables are known:
+
+ READY=1 Tells systemd that daemon startup is finished (only
+ relevant for services of Type=notify). The passed
+ argument is a boolean "1" or "0". Since there is
+ little value in signalling non-readiness the only
+ value daemons should send is "READY=1".
+
+ STATUS=... Passes a single-line status string back to systemd
+ that describes the daemon state. This is free-from
+ and can be used for various purposes: general state
+ feedback, fsck-like programs could pass completion
+ percentages and failing programs could pass a human
+ readable error message. Example: "STATUS=Completed
+ 66% of file system check..."
+
+ ERRNO=... If a daemon fails, the errno-style error code,
+ formatted as string. Example: "ERRNO=2" for ENOENT.
+
+ BUSERROR=... If a daemon fails, the D-Bus error-style error
+ code. Example: "BUSERROR=org.freedesktop.DBus.Error.TimedOut"
+
+ MAINPID=... The main pid of a daemon, in case systemd did not
+ fork off the process itself. Example: "MAINPID=4711"
+
+ Daemons can choose to send additional variables. However, it is
+ recommened to prefix variable names not listed above with X_.
+
+ Returns a negative errno-style error code on failure. Returns > 0
+ if systemd could be notified, 0 if it couldn't possibly because
+ systemd is not running.
+
+ Example: When a daemon finished starting up, it could issue this
+ call to notify systemd about it:
+
+ sd_notify(0, "READY=1");
+
+ See sd_notifyf() for more complete examples.
+
+ See sd_notify(3) for more information.
+*/
+int sd_notify(int unset_environment, const char *state) _sd_hidden_;
+
+/*
+ Similar to sd_notify() but takes a format string.
+
+ Example 1: A daemon could send the following after initialization:
+
+ sd_notifyf(0, "READY=1\n"
+ "STATUS=Processing requests...\n"
+ "MAINPID=%lu",
+ (unsigned long) getpid());
+
+ Example 2: A daemon could send the following shortly before
+ exiting, on failure:
+
+ sd_notifyf(0, "STATUS=Failed to start up: %s\n"
+ "ERRNO=%i",
+ strerror(errno),
+ errno);
+
+ See sd_notifyf(3) for more information.
+*/
+int sd_notifyf(int unset_environment, const char *format, ...) _sd_printf_attr_(2,3) _sd_hidden_;
+
+/*
+ Returns > 0 if the system was booted with systemd. Returns < 0 on
+ error. Returns 0 if the system was not booted with systemd. Note
+ that all of the functions above handle non-systemd boots just
+ fine. You should NOT protect them with a call to this function. Also
+ note that this function checks whether the system, not the user
+ session is controlled by systemd. However the functions above work
+ for both session and system services.
+
+ See sd_booted(3) for more information.
+*/
+int sd_booted(void) _sd_hidden_;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/runtime/srUtils.h b/runtime/srUtils.h
index 16766312..c4f73e16 100644
--- a/runtime/srUtils.h
+++ b/runtime/srUtils.h
@@ -110,30 +110,17 @@ rsRetVal getFileSize(uchar *pszName, off_t *pSize);
/* some useful constants */
#define DEFVARS_mutexProtection\
- int iCancelStateSave; \
int bLockedOpIsLocked=0
#define BEGIN_MTX_PROTECTED_OPERATIONS(mut, bMustLock) \
if(bMustLock == LOCK_MUTEX) { \
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \
d_pthread_mutex_lock(mut); \
+ assert(bLockedOpIsLocked == 0); \
bLockedOpIsLocked = 1; \
}
#define END_MTX_PROTECTED_OPERATIONS(mut) \
if(bLockedOpIsLocked) { \
d_pthread_mutex_unlock(mut); \
- pthread_setcancelstate(iCancelStateSave, NULL); \
+ bLockedOpIsLocked = 0; \
}
-/* The unconditional versions of the macro always lock the mutex. They are preferred in
- * complex scenarios, where the simple ones might get mixed up by multiple calls.
- */
-#define DEFVARS_mutexProtection_uncond\
- int iCancelStateSave
-#define BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \
- d_pthread_mutex_lock(mut);
-#define END_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \
- d_pthread_mutex_unlock(mut); \
- pthread_setcancelstate(iCancelStateSave, NULL);
-
#endif
diff --git a/runtime/srutils.c b/runtime/srutils.c
index 1452c9b7..d357cd77 100644
--- a/runtime/srutils.c
+++ b/runtime/srutils.c
@@ -46,6 +46,9 @@
#include "srUtils.h"
#include "obj.h"
+#if _POSIX_TIMERS <= 0
+#include <sys/time.h>
+#endif
/* here we host some syslog specific names. There currently is no better place
* to do it, but over here is also not ideal... -- rgerhards, 2008-02-14
@@ -158,7 +161,7 @@ uchar *srUtilStrDup(uchar *pOld, size_t len)
assert(pOld != NULL);
- if((pNew = malloc(len + 1)) != NULL)
+ if((pNew = MALLOC(len + 1)) != NULL)
memcpy(pNew, pOld, len + 1);
return pNew;
@@ -197,7 +200,7 @@ int makeFileParentDirs(uchar *szFile, size_t lenFile, mode_t mode,
assert(lenFile > 0);
len = lenFile + 1; /* add one for '\0'-byte */
- if((pszWork = malloc(sizeof(uchar) * len)) == NULL)
+ if((pszWork = MALLOC(sizeof(uchar) * len)) == NULL)
return -1;
memcpy(pszWork, szFile, len);
for(p = pszWork+1 ; *p ; p++)
@@ -346,7 +349,7 @@ rsRetVal genFileName(uchar **ppName, uchar *pDirName, size_t lenDirName, uchar *
}
lenName = lenDirName + 1 + lenFName + lenBuf + 1; /* last +1 for \0 char! */
- if((pName = malloc(sizeof(uchar) * lenName)) == NULL)
+ if((pName = MALLOC(sizeof(uchar) * lenName)) == NULL)
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
/* got memory, now construct string */
@@ -392,10 +395,22 @@ int getNumberDigits(long lNum)
rsRetVal
timeoutComp(struct timespec *pt, long iTimeout)
{
+# if _POSIX_TIMERS <= 0
+ struct timeval tv;
+# endif
+
BEGINfunc
assert(pt != NULL);
/* compute timeout */
+
+# if _POSIX_TIMERS > 0
+ /* this is the "regular" code */
clock_gettime(CLOCK_REALTIME, pt);
+# else
+ gettimeofday(&tv, NULL);
+ pt->tv_sec = tv.tv_sec;
+ pt->tv_nsec = tv.tv_usec * 1000;
+# endif
pt->tv_sec += iTimeout / 1000;
pt->tv_nsec += (iTimeout % 1000) * 1000000; /* think INTEGER arithmetic! */
if(pt->tv_nsec > 999999999) { /* overrun? */
@@ -417,11 +432,21 @@ timeoutVal(struct timespec *pt)
{
struct timespec t;
long iTimeout;
- BEGINfunc
+# if _POSIX_TIMERS <= 0
+ struct timeval tv;
+# endif
+ BEGINfunc
assert(pt != NULL);
/* compute timeout */
+# if _POSIX_TIMERS > 0
+ /* this is the "regular" code */
clock_gettime(CLOCK_REALTIME, &t);
+# else
+ gettimeofday(&tv, NULL);
+ t.tv_sec = tv.tv_sec;
+ t.tv_nsec = tv.tv_usec * 1000;
+# endif
iTimeout = (pt->tv_nsec - t.tv_nsec) / 1000000;
iTimeout += (pt->tv_sec - t.tv_sec) * 1000;
diff --git a/runtime/statsobj.c b/runtime/statsobj.c
new file mode 100644
index 00000000..e1a89cf4
--- /dev/null
+++ b/runtime/statsobj.c
@@ -0,0 +1,315 @@
+/* The statsobj object.
+ *
+ * This object provides a statistics-gathering facility inside rsyslog. This
+ * functionality will be pragmatically implemented and extended.
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+
+#include "config.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <pthread.h>
+#include <errno.h>
+#include <assert.h>
+
+#include "rsyslog.h"
+#include "unicode-helper.h"
+#include "obj.h"
+#include "statsobj.h"
+#include "sysvar.h"
+#include "srUtils.h"
+#include "stringbuf.h"
+
+
+/* externally-visiable data (see statsobj.h for explanation) */
+int GatherStats = 0;
+
+/* static data */
+DEFobjStaticHelpers
+
+/* doubly linked list of stats objects. Object is automatically linked to it
+ * upon construction. Enqueue always happens at the front (simplifies logic).
+ */
+static statsobj_t *objRoot = NULL;
+static statsobj_t *objLast = NULL;
+
+static pthread_mutex_t mutStats;
+
+/* ------------------------------ statsobj linked list maintenance ------------------------------ */
+
+static inline void
+addToObjList(statsobj_t *pThis)
+{
+ pthread_mutex_lock(&mutStats);
+ pThis->prev = objLast;
+ if(objLast != NULL)
+ objLast->next = pThis;
+ objLast = pThis;
+ if(objRoot == NULL)
+ objRoot = pThis;
+ pthread_mutex_unlock(&mutStats);
+}
+
+
+static inline void
+removeFromObjList(statsobj_t *pThis)
+{
+ pthread_mutex_lock(&mutStats);
+ if(pThis->prev != NULL)
+ pThis->prev->next = pThis->next;
+ if(pThis->next != NULL)
+ pThis->next->prev = pThis->prev;
+ if(objLast == pThis)
+ objLast = pThis->prev;
+ if(objRoot == pThis)
+ objRoot = pThis->next;
+ pthread_mutex_unlock(&mutStats);
+}
+
+
+static inline void
+addCtrToList(statsobj_t *pThis, ctr_t *pCtr)
+{
+ pthread_mutex_lock(&pThis->mutCtr);
+ pCtr->prev = pThis->ctrLast;
+ if(pThis->ctrLast != NULL)
+ pThis->ctrLast->next = pCtr;
+ pThis->ctrLast = pCtr;
+ if(pThis->ctrRoot == NULL)
+ pThis->ctrRoot = pCtr;
+ pthread_mutex_unlock(&pThis->mutCtr);
+}
+
+/* ------------------------------ methods ------------------------------ */
+
+
+/* Standard-Constructor
+ */
+BEGINobjConstruct(statsobj) /* be sure to specify the object type also in END macro! */
+ pthread_mutex_init(&pThis->mutCtr, NULL);
+ pThis->ctrLast = NULL;
+ pThis->ctrRoot = NULL;
+ENDobjConstruct(statsobj)
+
+
+/* ConstructionFinalizer
+ */
+static rsRetVal
+statsobjConstructFinalize(statsobj_t *pThis)
+{
+ DEFiRet;
+ ISOBJ_TYPE_assert(pThis, statsobj);
+ addToObjList(pThis);
+ RETiRet;
+}
+
+
+/* set name. Note that we make our own copy of the memory, caller is
+ * responsible to free up name it passes in (if required).
+ */
+static rsRetVal
+setName(statsobj_t *pThis, uchar *name)
+{
+ DEFiRet;
+ CHKmalloc(pThis->name = ustrdup(name));
+finalize_it:
+ RETiRet;
+}
+
+
+/* add a counter to an object
+ * ctrName is duplicated, caller must free it if requried
+ */
+static rsRetVal
+addCounter(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr)
+{
+ ctr_t *ctr;
+ DEFiRet;
+
+ CHKmalloc(ctr = malloc(sizeof(ctr_t)));
+ ctr->next = NULL;
+ ctr->prev = NULL;
+ CHKmalloc(ctr->name = ustrdup(ctrName));
+ ctr->ctrType = ctrType;
+ switch(ctrType) {
+ case ctrType_IntCtr:
+ ctr->val.pIntCtr = (intctr_t*) pCtr;
+ break;
+ case ctrType_Int:
+ ctr->val.pInt = (int*) pCtr;
+ break;
+ }
+ addCtrToList(pThis, ctr);
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* get all the object's countes together with object name as one line.
+ */
+static rsRetVal
+getStatsLine(statsobj_t *pThis, cstr_t **ppcstr)
+{
+ cstr_t *pcstr;
+ ctr_t *pCtr;
+ DEFiRet;
+
+ CHKiRet(cstrConstruct(&pcstr));
+ rsCStrAppendStr(pcstr, pThis->name);
+ rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT(": "), 2);
+
+ /* now add all counters to this line */
+ pthread_mutex_lock(&pThis->mutCtr);
+ for(pCtr = pThis->ctrRoot ; pCtr != NULL ; pCtr = pCtr->next) {
+ rsCStrAppendStr(pcstr, pCtr->name);
+ cstrAppendChar(pcstr, '=');
+ switch(pCtr->ctrType) {
+ case ctrType_IntCtr:
+ rsCStrAppendInt(pcstr, *(pCtr->val.pIntCtr)); // TODO: OK?????
+ break;
+ case ctrType_Int:
+ rsCStrAppendInt(pcstr, *(pCtr->val.pInt));
+ break;
+ }
+ cstrAppendChar(pcstr, ' ');
+ }
+ pthread_mutex_unlock(&pThis->mutCtr);
+
+ CHKiRet(cstrFinalize(pcstr));
+ *ppcstr = pcstr;
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* this function can be used to obtain all stats lines. In this case,
+ * a callback must be provided. This module than iterates over all objects and
+ * submits each stats line to the callback. The callback has two parameters:
+ * the first one is a caller-provided void*, the second one the cstr_t with the
+ * line. If the callback reports an error, processing is stopped.
+ */
+static rsRetVal
+getAllStatsLines(rsRetVal(*cb)(void*, cstr_t*), void *usrptr)
+{
+ statsobj_t *o;
+ cstr_t *cstr;
+ DEFiRet;
+
+ for(o = objRoot ; o != NULL ; o = o->next) {
+ CHKiRet(getStatsLine(o, &cstr));
+ CHKiRet(cb(usrptr, cstr));
+ rsCStrDestruct(&cstr);
+ }
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* Enable statistics gathering. currently there is no function to disable it
+ * again, as this is right now not needed.
+ */
+static rsRetVal
+enableStats()
+{
+ GatherStats = 1;
+ return RS_RET_OK;
+}
+
+
+/* destructor for the statsobj object */
+BEGINobjDestruct(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */
+ ctr_t *ctr, *ctrToDel;
+CODESTARTobjDestruct(statsobj)
+ removeFromObjList(pThis);
+
+ /* destruct counters */
+ ctr = pThis->ctrRoot;
+ while(ctr != NULL) {
+ ctrToDel = ctr;
+ ctr = ctr->next;
+ free(ctrToDel->name);
+ free(ctrToDel);
+ }
+
+ pthread_mutex_destroy(&pThis->mutCtr);
+ free(pThis->name);
+ENDobjDestruct(statsobj)
+
+
+/* debugprint for the statsobj object */
+BEGINobjDebugPrint(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */
+CODESTARTobjDebugPrint(statsobj)
+ dbgoprint((obj_t*) pThis, "statsobj object, currently no state info available\n");
+ENDobjDebugPrint(statsobj)
+
+
+/* queryInterface function
+ */
+BEGINobjQueryInterface(statsobj)
+CODESTARTobjQueryInterface(statsobj)
+ if(pIf->ifVersion != statsobjCURR_IF_VERSION) { /* check for current version, increment on each change */
+ ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED);
+ }
+
+ /* ok, we have the right interface, so let's fill it
+ * Please note that we may also do some backwards-compatibility
+ * work here (if we can support an older interface version - that,
+ * of course, also affects the "if" above).
+ */
+ pIf->Construct = statsobjConstruct;
+ pIf->ConstructFinalize = statsobjConstructFinalize;
+ pIf->Destruct = statsobjDestruct;
+ pIf->DebugPrint = statsobjDebugPrint;
+ pIf->SetName = setName;
+ pIf->GetStatsLine = getStatsLine;
+ pIf->GetAllStatsLines = getAllStatsLines;
+ pIf->AddCounter = addCounter;
+ pIf->EnableStats = enableStats;
+finalize_it:
+ENDobjQueryInterface(statsobj)
+
+
+/* Initialize the statsobj class. Must be called as the very first method
+ * before anything else is called inside this class.
+ */
+BEGINAbstractObjClassInit(statsobj, 1, OBJ_IS_CORE_MODULE) /* class, version */
+ /* request objects we use */
+
+ /* set our own handlers */
+ OBJSetMethodHandler(objMethod_DEBUGPRINT, statsobjDebugPrint);
+ OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, statsobjConstructFinalize);
+
+ /* init other data items */
+ pthread_mutex_init(&mutStats, NULL);
+
+ENDObjClassInit(statsobj)
+
+/* Exit the class.
+ */
+BEGINObjClassExit(statsobj, OBJ_IS_CORE_MODULE) /* class, version */
+ /* release objects we no longer need */
+ pthread_mutex_destroy(&mutStats);
+ENDObjClassExit(statsobj)
diff --git a/runtime/statsobj.h b/runtime/statsobj.h
new file mode 100644
index 00000000..44c26bea
--- /dev/null
+++ b/runtime/statsobj.h
@@ -0,0 +1,124 @@
+/* The statsobj object.
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#ifndef INCLUDED_STATSOBJ_H
+#define INCLUDED_STATSOBJ_H
+
+#include "atomic.h"
+
+/* The following data item is somewhat dirty, in that it does not follow
+ * our usual object calling conventions. However, much like with "Debug", we
+ * do this to gain speed. If we finally come to a platform that does not
+ * provide resolution of names for dynamically loaded modules, we need to find
+ * a work-around, but until then, we use the direct access.
+ * If set to 0, statistics are not gathered, otherwise they are.
+ */
+extern int GatherStats;
+
+/* our basic counter type -- need 32 bit on 32 bit platform.
+ * IMPORTANT: this type *MUST* be supported by atomic instructions!
+ */
+typedef uint64 intctr_t;
+
+/* counter types */
+typedef enum statsCtrType_e {
+ ctrType_IntCtr,
+ ctrType_Int
+} statsCtrType_t;
+
+
+/* helper entity, the counter */
+typedef struct ctr_s {
+ uchar *name;
+ statsCtrType_t ctrType;
+ union {
+ intctr_t *pIntCtr;
+ int *pInt;
+ } val;
+ struct ctr_s *next, *prev;
+} ctr_t;
+
+/* the statsobj object */
+struct statsobj_s {
+ BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */
+ uchar *name;
+ pthread_mutex_t mutCtr; /* to guard counter linked-list ops */
+ ctr_t *ctrRoot; /* doubly-linked list of statsobj counters */
+ ctr_t *ctrLast;
+ /* used to link ourselves together */
+ statsobj_t *prev;
+ statsobj_t *next;
+};
+
+
+/* interfaces */
+BEGINinterface(statsobj) /* name must also be changed in ENDinterface macro! */
+ INTERFACEObjDebugPrint(statsobj);
+ rsRetVal (*Construct)(statsobj_t **ppThis);
+ rsRetVal (*ConstructFinalize)(statsobj_t *pThis);
+ rsRetVal (*Destruct)(statsobj_t **ppThis);
+ rsRetVal (*SetName)(statsobj_t *pThis, uchar *name);
+ rsRetVal (*GetStatsLine)(statsobj_t *pThis, cstr_t **ppcstr);
+ rsRetVal (*GetAllStatsLines)(rsRetVal(*cb)(void*, cstr_t*), void *usrptr);
+ rsRetVal (*AddCounter)(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr);
+ rsRetVal (*EnableStats)(void);
+ENDinterface(statsobj)
+#define statsobjCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
+
+
+/* prototypes */
+PROTOTYPEObj(statsobj);
+
+
+/* macros to handle stats counters
+ * These are to be used by "counter providers". Note that we MUST
+ * specify the mutex name, even though at first it looks like it
+ * could be automatically be generated via e.g. "mut##ctr".
+ * Unfortunately, this does not work if counter is e.g. "pThis->ctr".
+ * So we decided, for clarity, to always insist on specifying the mutex
+ * name (after all, it's just a few more keystrokes...).
+ */
+#define STATSCOUNTER_DEF(ctr, mut) \
+ intctr_t ctr; \
+ DEF_ATOMIC_HELPER_MUT64(mut);
+
+#define STATSCOUNTER_INIT(ctr, mut) \
+ INIT_ATOMIC_HELPER_MUT64(mut); \
+ ctr = 0;
+
+#define STATSCOUNTER_INC(ctr, mut) \
+ if(GatherStats) \
+ ATOMIC_INC_uint64(&ctr, &mut);
+
+#define STATSCOUNTER_DEC(ctr, mut) \
+ if(GatherStats) \
+ ATOMIC_DEC_uint64(&ctr, mut);
+
+/* the next macro works only if the variable is already guarded
+ * by mutex (or the users risks a wrong result). It is assumed
+ * that there are not concurrent operations that modify the counter.
+ */
+#define STATSCOUNTER_SETMAX_NOMUT(ctr, newmax) \
+ if(GatherStats && ((newmax) > (ctr))) \
+ ctr = newmax;
+
+#endif /* #ifndef INCLUDED_STATSOBJ_H */
diff --git a/runtime/stream.c b/runtime/stream.c
index 44b24ee2..6b88d3f4 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -269,7 +269,7 @@ static rsRetVal strmOpenFile(strm_t *pThis)
pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits));
} else {
if(pThis->pszDir == NULL) {
- if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL)
+ if((pThis->pszCurrFName = ustrdup(pThis->pszFName)) == NULL)
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
} else {
CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir,
@@ -573,39 +573,100 @@ static rsRetVal strmUnreadChar(strm_t *pThis, uchar c)
return RS_RET_OK;
}
-
-/* read a line from a strm file. A line is terminated by LF. The LF is read, but it
- * is not returned in the buffer (it is discared). The caller is responsible for
- * destruction of the returned CStr object! -- rgerhards, 2008-01-07
- * rgerhards, 2008-03-27: I now use the ppCStr directly, without any interim
- * string pointer. The reason is that this function my be called by inputs, which
- * are pthread_killed() upon termination. So if we use their native pointer, they
- * can cleanup (but only then).
+/* read a 'paragraph' from a strm file.
+ * A paragraph may be terminated by a LF, by a LFLF, or by LF<not whitespace> depending on the option set.
+ * The termination LF characters are read, but are
+ * not returned in the buffer (it is discared). The caller is responsible for
+ * destruction of the returned CStr object! -- dlang 2010-12-13
*/
static rsRetVal
-strmReadLine(strm_t *pThis, cstr_t **ppCStr)
+strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode)
{
- DEFiRet;
- uchar c;
-
- ASSERT(pThis != NULL);
- ASSERT(ppCStr != NULL);
-
- CHKiRet(cstrConstruct(ppCStr));
-
- /* now read the line */
- CHKiRet(strmReadChar(pThis, &c));
- while(c != '\n') {
- CHKiRet(cstrAppendChar(*ppCStr, c));
- CHKiRet(strmReadChar(pThis, &c));
+ /* mode = 0 single line mode (equivalent to ReadLine)
+ * mode = 1 LFLF mode (paragraph, blank line between entries)
+ * mode = 2 LF <not whitespace> mode, a log line starts at the beginning of a line, but following lines that are indented are part of the same log entry
+ * This modal interface is not nearly as flexible as being able to define a regex for when a new record starts, but it's also not nearly as hard (or as slow) to implement
+ */
+ DEFiRet;
+ uchar c;
+ uchar finished;
+
+ ASSERT(pThis != NULL);
+ ASSERT(ppCStr != NULL);
+
+ CHKiRet(cstrConstruct(ppCStr));
+
+ /* now read the line */
+ CHKiRet(strmReadChar(pThis, &c));
+ if (mode == 0){
+ while(c != '\n') {
+ CHKiRet(cstrAppendChar(*ppCStr, c));
+ CHKiRet(strmReadChar(pThis, &c));
+ }
+ CHKiRet(cstrFinalize(*ppCStr));
+ }
+ if (mode == 1){
+ finished=0;
+ while(finished == 0){
+ if(c != '\n') {
+ CHKiRet(cstrAppendChar(*ppCStr, c));
+ CHKiRet(strmReadChar(pThis, &c));
+ } else {
+ if ((((*ppCStr)->iStrLen) > 0) ){
+ if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] == '\n'){
+ rsCStrTruncate(*ppCStr,1); /* remove the prior newline */
+ finished=1;
+ } else {
+ CHKiRet(cstrAppendChar(*ppCStr, c));
+ CHKiRet(strmReadChar(pThis, &c));
+ }
+ } else {
+ finished=1; /* this is a blank line, a \n with nothing since the last complete record */
+ }
+ }
+ }
+ CHKiRet(cstrFinalize(*ppCStr));
+ }
+ if (mode == 2){
+ /* indented follow-up lines */
+ finished=0;
+ while(finished == 0){
+ if ((*ppCStr)->iStrLen == 0){
+ if(c != '\n') {
+ /* nothing in the buffer, and it's not a newline, add it to the buffer */
+ CHKiRet(cstrAppendChar(*ppCStr, c));
+ CHKiRet(strmReadChar(pThis, &c));
+ } else {
+ finished=1; /* this is a blank line, a \n with nothing since the last complete record */
+ }
+ } else {
+ if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] != '\n'){
+ /* not the first character after a newline, add it to the buffer */
+ CHKiRet(cstrAppendChar(*ppCStr, c));
+ CHKiRet(strmReadChar(pThis, &c));
+ } else {
+ if ((c == ' ') || (c == '\t')){
+ CHKiRet(cstrAppendChar(*ppCStr, c));
+ CHKiRet(strmReadChar(pThis, &c));
+ } else {
+ /* clean things up by putting the character we just read back into
+ * the input buffer and removing the LF character that is currently at the
+ * end of the output string */
+ CHKiRet(strmUnreadChar(pThis, c));
+ rsCStrTruncate(*ppCStr,1);
+ finished=1;
+ }
+ }
+ }
+ }
+ CHKiRet(cstrFinalize(*ppCStr));
}
- CHKiRet(cstrFinalize(*ppCStr));
finalize_it:
- if(iRet != RS_RET_OK && *ppCStr != NULL)
- cstrDestruct(ppCStr);
+ if(iRet != RS_RET_OK && *ppCStr != NULL)
+ cstrDestruct(ppCStr);
- RETiRet;
+ RETiRet;
}
@@ -645,7 +706,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis)
* to make sure we can write out everything with a SINGLE api call!
* We add another 128 bytes to take care of the gzip header and "all eventualities".
*/
- CHKmalloc(pThis->pZipBuf = (Bytef*) malloc(sizeof(uchar) * (pThis->sIOBufSize + 128)));
+ CHKmalloc(pThis->pZipBuf = (Bytef*) MALLOC(sizeof(uchar) * (pThis->sIOBufSize + 128)));
}
}
@@ -677,15 +738,21 @@ static rsRetVal strmConstructFinalize(strm_t *pThis)
pthread_cond_init(&pThis->isEmpty, 0);
pThis->iCnt = pThis->iEnq = pThis->iDeq = 0;
for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) {
- CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize));
+ CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize));
}
pThis->pIOBuf = pThis->asyncBuf[0].pBuf;
pThis->bStopWriter = 0;
- if(pthread_create(&pThis->writerThreadID, NULL, asyncWriterThread, pThis) != 0)
+ if(pthread_create(&pThis->writerThreadID,
+#ifdef HAVE_PTHREAD_SETSCHEDPARAM
+ &default_thread_attr,
+#else
+ NULL,
+#endif
+ asyncWriterThread, pThis) != 0)
DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis);
} else {
/* we work synchronously, so we need to alloc a fixed pIOBuf */
- CHKmalloc(pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize));
+ CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize));
}
finalize_it:
@@ -943,7 +1010,7 @@ asyncWriterThread(void *pPtr)
{
int iDeq;
struct timespec t;
- bool bTimedOut = 0;
+ sbool bTimedOut = 0;
strm_t *pThis = (strm_t*) pPtr;
ISOBJ_TYPE_assert(pThis, strm);
@@ -956,7 +1023,6 @@ asyncWriterThread(void *pPtr)
while(1) { /* loop broken inside */
d_pthread_mutex_lock(&pThis->mut);
-dbgprintf("XXX: asyncWriterThread iterating %s\n", pThis->pszFName);
while(pThis->iCnt == 0) {
if(pThis->bStopWriter) {
pthread_cond_broadcast(&pThis->isEmpty);
@@ -970,9 +1036,8 @@ dbgprintf("XXX: asyncWriterThread iterating %s\n", pThis->pszFName);
continue; /* now we should have data */
}
bTimedOut = 0;
- timeoutComp(&t, pThis->iFlushInterval * 2000); /* *1000 millisconds */ // TODO: check the 2000?!?
+ timeoutComp(&t, pThis->iFlushInterval * 1000); /* *1000 millisconds */
if(pThis->bDoTimedWait) {
-dbgprintf("asyncWriter thread going to timeout sleep\n");
if(pthread_cond_timedwait(&pThis->notEmpty, &pThis->mut, &t) != 0) {
int err = errno;
if(err == ETIMEDOUT) {
@@ -986,16 +1051,13 @@ dbgprintf("asyncWriter thread going to timeout sleep\n");
}
}
} else {
-dbgprintf("asyncWriter thread going to eternal sleep\n");
d_pthread_cond_wait(&pThis->notEmpty, &pThis->mut);
}
-dbgprintf("asyncWriter woke up\n");
}
bTimedOut = 0; /* we may have timed out, but there *is* work to do... */
iDeq = pThis->iDeq++ % STREAM_ASYNC_NUMBUFS;
-dbgprintf("asyncWriter writes data\n");
doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf);
// TODO: error check????? 2009-07-06
@@ -1110,7 +1172,7 @@ doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf)
{
z_stream zstrm;
int zRet; /* zlib return state */
- bool bzInitDone = FALSE;
+ sbool bzInitDone = FALSE;
DEFiRet;
assert(pThis != NULL);
assert(pBuf != NULL);
@@ -1214,16 +1276,18 @@ static rsRetVal strmSeek(strm_t *pThis, off64_t offs)
ISOBJ_TYPE_assert(pThis, strm);
- if(pThis->fd == -1)
- strmOpenFile(pThis);
- else
- strmFlushInternal(pThis);
+ if(pThis->fd == -1) {
+ CHKiRet(strmOpenFile(pThis));
+ } else {
+ CHKiRet(strmFlushInternal(pThis));
+ }
long long i;
DBGOPRINT((obj_t*) pThis, "file %d seek, pos %llu\n", pThis->fd, (long long unsigned) offs);
i = lseek64(pThis->fd, offs, SEEK_SET); // TODO: check error!
pThis->iCurrOffs = offs; /* we are now at *this* offset */
pThis->iBufPtr = 0; /* buffer invalidated */
+finalize_it:
RETiRet;
}
@@ -1404,7 +1468,7 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName)
if(pThis->pszFName != NULL)
free(pThis->pszFName);
- if((pThis->pszFName = malloc(sizeof(uchar) * (iLenName + 1))) == NULL)
+ if((pThis->pszFName = MALLOC(sizeof(uchar) * (iLenName + 1))) == NULL)
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */
@@ -1431,7 +1495,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir)
if(iLenDir < 1)
ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING);
- CHKmalloc(pThis->pszDir = malloc(sizeof(uchar) * iLenDir + 1));
+ CHKmalloc(pThis->pszDir = MALLOC(sizeof(uchar) * (iLenDir + 1)));
memcpy(pThis->pszDir, pszDir, iLenDir + 1); /* always think about the \0! */
pThis->lenDir = iLenDir;
@@ -1529,6 +1593,46 @@ finalize_it:
}
+/* duplicate a stream object excluding dynamic properties. This function is
+ * primarily meant to provide a duplicate that later on can be used to access
+ * the data. This is needed, for example, for a restart of the disk queue.
+ * Note that ConstructFinalize() is NOT called. So our caller may change some
+ * properties before finalizing things.
+ * rgerhards, 2009-05-26
+ */
+rsRetVal
+strmDup(strm_t *pThis, strm_t **ppNew)
+{
+ strm_t *pNew = NULL;
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, strm);
+ assert(ppNew != NULL);
+
+ CHKiRet(strmConstruct(&pNew));
+ pNew->sType = pThis->sType;
+ pNew->iCurrFNum = pThis->iCurrFNum;
+ CHKmalloc(pNew->pszFName = ustrdup(pThis->pszFName));
+ pNew->lenFName = pThis->lenFName;
+ CHKmalloc(pNew->pszDir = ustrdup(pThis->pszDir));
+ pNew->lenDir = pThis->lenDir;
+ pNew->tOperationsMode = pThis->tOperationsMode;
+ pNew->tOpenMode = pThis->tOpenMode;
+ pNew->iMaxFileSize = pThis->iMaxFileSize;
+ pNew->iMaxFiles = pThis->iMaxFiles;
+ pNew->iFileNumDigits = pThis->iFileNumDigits;
+ pNew->bDeleteOnClose = pThis->bDeleteOnClose;
+ pNew->iCurrOffs = pThis->iCurrOffs;
+
+ *ppNew = pNew;
+ pNew = NULL;
+
+finalize_it:
+ if(pNew != NULL)
+ strmDestruct(&pNew);
+
+ RETiRet;
+}
/* set a user write-counter. This counter is initialized to zero and
* receives the number of bytes written. It is accurate only after a
@@ -1644,6 +1748,7 @@ CODESTARTobjQueryInterface(strm)
pIf->RecordEnd = strmRecordEnd;
pIf->Serialize = strmSerialize;
pIf->GetCurrOffset = strmGetCurrOffset;
+ pIf->Dup = strmDup;
pIf->SetWCntr = strmSetWCntr;
/* set methods */
pIf->SetbDeleteOnClose = strmSetbDeleteOnClose;
diff --git a/runtime/stream.h b/runtime/stream.h
index 369d5a0f..60c68cb2 100644
--- a/runtime/stream.h
+++ b/runtime/stream.h
@@ -102,12 +102,12 @@ typedef struct strm_s {
int64 iMaxFileSize;/* maximum size a file may grow to */
int iMaxFiles; /* maximum number of files if a circular mode is in use */
int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */
- bool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */
+ sbool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */
int64 iCurrOffs;/* current offset */
int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */
/* dynamic properties, valid only during file open, not to be persistet */
- bool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */
- bool bSync; /* sync this file after every write? */
+ sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */
+ sbool bSync; /* sync this file after every write? */
size_t sIOBufSize;/* size of IO buffer */
uchar *pszDir; /* Directory */
int lenDir;
@@ -118,13 +118,13 @@ typedef struct strm_s {
size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */
size_t iBufPtr; /* pointer into current buffer */
int iUngetC; /* char set via UngetChar() call or -1 if none set */
- bool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */
+ sbool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */
int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */
Bytef *pZipBuf;
/* support for async flush procesing */
- bool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */
- bool bStopWriter; /* shall writer thread terminate? */
- bool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */
+ sbool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */
+ sbool bStopWriter; /* shall writer thread terminate? */
+ sbool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */
int iFlushInterval; /* flush in which interval - 0, no flushing */
apc_id_t apcID; /* id of current Apc request (used for cancelling) */
pthread_mutex_t mut;/* mutex for flush in async mode */
@@ -143,7 +143,7 @@ typedef struct strm_s {
/* support for omfile size-limiting commands, special counters, NOT persisted! */
off_t iSizeLimit; /* file size limit, 0 = no limit */
uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */
- bool bIsTTY; /* is this a tty file? */
+ sbool bIsTTY; /* is this a tty file? */
} strm_t;
@@ -156,7 +156,6 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */
rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName);
rsRetVal (*ReadChar)(strm_t *pThis, uchar *pC);
rsRetVal (*UnreadChar)(strm_t *pThis, uchar c);
- rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr);
rsRetVal (*SeekCurrOffs)(strm_t *pThis);
rsRetVal (*Write)(strm_t *pThis, uchar *pBuf, size_t lenBuf);
rsRetVal (*WriteChar)(strm_t *pThis, uchar c);
@@ -169,6 +168,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */
rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm);
rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs);
rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt);
+ rsRetVal (*Dup)(strm_t *pThis, strm_t **ppNew);
INTERFACEpropSetMeth(strm, bDeleteOnClose, int);
INTERFACEpropSetMeth(strm, iMaxFileSize, int);
INTERFACEpropSetMeth(strm, iMaxFiles, int);
@@ -182,8 +182,10 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */
INTERFACEpropSetMeth(strm, iSizeLimit, off_t);
INTERFACEpropSetMeth(strm, iFlushInterval, int);
INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*);
+ /* v6 added */
+ rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr, int mode);
ENDinterface(strm)
-#define strmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */
+#define strmCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */
/* prototypes */
diff --git a/runtime/strgen.c b/runtime/strgen.c
new file mode 100644
index 00000000..46be1236
--- /dev/null
+++ b/runtime/strgen.c
@@ -0,0 +1,279 @@
+/* strgen.c
+ * Module to handle string generators. These are C modules that receive
+ * the message object and return a custom-built string. The primary purpose
+ * for their existance is performance -- they do the same as template strings, but
+ * potentially faster (if well implmented).
+ *
+ * Module begun 2010-06-01 by Rainer Gerhards
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#include "config.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "rsyslog.h"
+#include "msg.h"
+#include "obj.h"
+#include "errmsg.h"
+#include "strgen.h"
+#include "ruleset.h"
+#include "unicode-helper.h"
+#include "cfsysline.h"
+
+/* definitions for objects we access */
+DEFobjStaticHelpers
+DEFobjCurrIf(glbl)
+DEFobjCurrIf(errmsg)
+DEFobjCurrIf(ruleset)
+
+/* static data */
+
+/* config data */
+
+/* This is the list of all strgens known to us.
+ * This is also used to unload all modules on shutdown.
+ */
+strgenList_t *pStrgenLstRoot = NULL;
+
+
+/* intialize (but NOT allocate) a strgen list. Primarily meant as a hook
+ * which can be used to extend the list in the future. So far, just sets
+ * it to NULL.
+ */
+static rsRetVal
+InitStrgenList(strgenList_t **pListRoot)
+{
+ *pListRoot = NULL;
+ return RS_RET_OK;
+}
+
+
+/* destruct a strgen list. The list elements are destroyed, but the strgen objects
+ * themselves are not modified. (That is done at a late stage during rsyslogd
+ * shutdown and need not be considered here.)
+ */
+static rsRetVal
+DestructStrgenList(strgenList_t **ppListRoot)
+{
+ strgenList_t *pStrgenLst;
+ strgenList_t *pStrgenLstDel;
+
+ pStrgenLst = *ppListRoot;
+ while(pStrgenLst != NULL) {
+ pStrgenLstDel = pStrgenLst;
+ pStrgenLst = pStrgenLst->pNext;
+ free(pStrgenLstDel);
+ }
+ *ppListRoot = NULL;
+ return RS_RET_OK;
+}
+
+
+/* Add a strgen to the list. We use a VERY simple and ineffcient algorithm,
+ * but it is employed only for a few milliseconds during config processing. So
+ * I prefer to keep it very simple and with simple data structures. Unfortunately,
+ * we need to preserve the order, but I don't like to add a tail pointer as that
+ * would require a container object. So I do the extra work to skip to the tail
+ * when adding elements...
+ */
+static rsRetVal
+AddStrgenToList(strgenList_t **ppListRoot, strgen_t *pStrgen)
+{
+ strgenList_t *pThis;
+ strgenList_t *pTail;
+ DEFiRet;
+
+ CHKmalloc(pThis = MALLOC(sizeof(strgenList_t)));
+ pThis->pStrgen = pStrgen;
+ pThis->pNext = NULL;
+
+ if(*ppListRoot == NULL) {
+ pThis->pNext = *ppListRoot;
+ *ppListRoot = pThis;
+ } else {
+ /* find tail first */
+ for(pTail = *ppListRoot ; pTail->pNext != NULL ; pTail = pTail->pNext)
+ /* just search, do nothing else */;
+ /* add at tail */
+ pTail->pNext = pThis;
+ }
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* find a strgen based on the provided name */
+static rsRetVal
+FindStrgen(strgen_t **ppStrgen, uchar *pName)
+{
+ strgenList_t *pThis;
+ DEFiRet;
+
+ for(pThis = pStrgenLstRoot ; pThis != NULL ; pThis = pThis->pNext) {
+ if(ustrcmp(pThis->pStrgen->pName, pName) == 0) {
+ *ppStrgen = pThis->pStrgen;
+ FINALIZE; /* found it, iRet still eq. OK! */
+ }
+ }
+
+ iRet = RS_RET_PARSER_NOT_FOUND;
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* --- END helper functions for strgen list handling --- */
+
+
+BEGINobjConstruct(strgen) /* be sure to specify the object type also in END macro! */
+ENDobjConstruct(strgen)
+
+/* ConstructionFinalizer. The most important chore is to add the strgen object
+ * to our global list of available strgens.
+ * rgerhards, 2009-11-03
+ */
+rsRetVal strgenConstructFinalize(strgen_t *pThis)
+{
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, strgen);
+ CHKiRet(AddStrgenToList(&pStrgenLstRoot, pThis));
+ DBGPRINTF("Strgen '%s' added to list of available strgens.\n", pThis->pName);
+
+finalize_it:
+ RETiRet;
+}
+
+BEGINobjDestruct(strgen) /* be sure to specify the object type also in END and CODESTART macros! */
+CODESTARTobjDestruct(strgen)
+ dbgprintf("destructing strgen '%s'\n", pThis->pName);
+ free(pThis->pName);
+ENDobjDestruct(strgen)
+
+/* set the strgen name - string is copied over, call can continue to use it,
+ * but must free it if desired.
+ */
+static rsRetVal
+SetName(strgen_t *pThis, uchar *name)
+{
+ DEFiRet;
+
+ ISOBJ_TYPE_assert(pThis, strgen);
+ assert(name != NULL);
+
+ if(pThis->pName != NULL) {
+ free(pThis->pName);
+ pThis->pName = NULL;
+ }
+
+ CHKmalloc(pThis->pName = ustrdup(name));
+
+finalize_it:
+ RETiRet;
+}
+
+
+/* set a pointer to "our" module. Note that no module
+ * pointer must already be set.
+ */
+static rsRetVal
+SetModPtr(strgen_t *pThis, modInfo_t *pMod)
+{
+ ISOBJ_TYPE_assert(pThis, strgen);
+ assert(pMod != NULL);
+ assert(pThis->pModule == NULL);
+ pThis->pModule = pMod;
+ return RS_RET_OK;
+}
+
+
+/* queryInterface function-- rgerhards, 2009-11-03
+ */
+BEGINobjQueryInterface(strgen)
+CODESTARTobjQueryInterface(strgen)
+ if(pIf->ifVersion != strgenCURR_IF_VERSION) { /* check for current version, increment on each change */
+ ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED);
+ }
+
+ /* ok, we have the right interface, so let's fill it
+ * Please note that we may also do some backwards-compatibility
+ * work here (if we can support an older interface version - that,
+ * of course, also affects the "if" above).
+ */
+ pIf->Construct = strgenConstruct;
+ pIf->ConstructFinalize = strgenConstructFinalize;
+ pIf->Destruct = strgenDestruct;
+ pIf->SetName = SetName;
+ pIf->SetModPtr = SetModPtr;
+ pIf->InitStrgenList = InitStrgenList;
+ pIf->DestructStrgenList = DestructStrgenList;
+ pIf->AddStrgenToList = AddStrgenToList;
+ pIf->FindStrgen = FindStrgen;
+finalize_it:
+ENDobjQueryInterface(strgen)
+
+
+/* This destroys the master strgenlist and all of its strgen entries. MUST only be
+ * done when the module is shut down. Strgen modules are NOT unloaded, rsyslog
+ * does that at a later stage for all dynamically loaded modules.
+ */
+static void
+destroyMasterStrgenList(void)
+{
+ strgenList_t *pStrgenLst;
+ strgenList_t *pStrgenLstDel;
+
+ pStrgenLst = pStrgenLstRoot;
+ while(pStrgenLst != NULL) {
+ strgenDestruct(&pStrgenLst->pStrgen);
+ pStrgenLstDel = pStrgenLst;
+ pStrgenLst = pStrgenLst->pNext;
+ free(pStrgenLstDel);
+ }
+}
+
+/* Exit our class.
+ * rgerhards, 2009-11-04
+ */
+BEGINObjClassExit(strgen, OBJ_IS_CORE_MODULE) /* class, version */
+ destroyMasterStrgenList();
+ objRelease(glbl, CORE_COMPONENT);
+ objRelease(errmsg, CORE_COMPONENT);
+ objRelease(ruleset, CORE_COMPONENT);
+ENDObjClassExit(strgen)
+
+
+/* Initialize the strgen class. Must be called as the very first method
+ * before anything else is called inside this class.
+ * rgerhards, 2009-11-02
+ */
+BEGINObjClassInit(strgen, 1, OBJ_IS_CORE_MODULE) /* class, version */
+ /* request objects we use */
+ CHKiRet(objUse(glbl, CORE_COMPONENT));
+ CHKiRet(objUse(errmsg, CORE_COMPONENT));
+ CHKiRet(objUse(ruleset, CORE_COMPONENT));
+ InitStrgenList(&pStrgenLstRoot);
+ENDObjClassInit(strgen)
+
diff --git a/runtime/strgen.h b/runtime/strgen.h
new file mode 100644
index 00000000..3819dccd
--- /dev/null
+++ b/runtime/strgen.h
@@ -0,0 +1,60 @@
+/* header for strgen.c
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#ifndef INCLUDED_STRGEN_H
+#define INCLUDED_STRGEN_H
+
+
+/* we create a small helper object, a list of strgens, that we can use to
+ * build a chain of them whereever this is needed.
+ */
+struct strgenList_s {
+ strgen_t *pStrgen;
+ strgenList_t *pNext;
+};
+
+
+/* the strgen object, a dummy because we have only static methods */
+struct strgen_s {
+ BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */
+ uchar *pName; /* name of this strgen */
+ modInfo_t *pModule; /* pointer to strgen's module */
+};
+
+/* interfaces */
+BEGINinterface(strgen) /* name must also be changed in ENDinterface macro! */
+ rsRetVal (*Construct)(strgen_t **ppThis);
+ rsRetVal (*ConstructFinalize)(strgen_t *pThis);
+ rsRetVal (*Destruct)(strgen_t **ppThis);
+ rsRetVal (*SetName)(strgen_t *pThis, uchar *name);
+ rsRetVal (*SetModPtr)(strgen_t *pThis, modInfo_t *pMod);
+ rsRetVal (*FindStrgen)(strgen_t **ppThis, uchar*name);
+ rsRetVal (*InitStrgenList)(strgenList_t **pListRoot);
+ rsRetVal (*DestructStrgenList)(strgenList_t **pListRoot);
+ rsRetVal (*AddStrgenToList)(strgenList_t **pListRoot, strgen_t *pStrgen);
+ENDinterface(strgen)
+#define strgenCURR_IF_VERSION 1 /* increment whenever you change the interface above! */
+
+
+/* prototypes */
+PROTOTYPEObj(strgen);
+
+#endif /* #ifndef INCLUDED_STRGEN_H */
diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c
index 8b2fe455..2b6815a4 100644
--- a/runtime/stringbuf.c
+++ b/runtime/stringbuf.c
@@ -90,7 +90,7 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz)
CHKiRet(rsCStrConstruct(&pThis));
pThis->iBufSize = pThis->iStrLen = strlen((char *) sz);
- if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) {
+ if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) {
RSFREEOBJ(pThis);
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
@@ -119,7 +119,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom)
CHKiRet(rsCStrConstruct(&pThis));
pThis->iBufSize = pThis->iStrLen = pFrom->iStrLen;
- if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) {
+ if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) {
RSFREEOBJ(pThis);
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
@@ -267,7 +267,7 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew)
pThis->pszBuf = NULL;
/* now save the new value */
- if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) {
+ if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) {
RSFREEOBJ(pThis);
return RS_RET_OUT_OF_MEMORY;
}
@@ -315,7 +315,7 @@ uchar* rsCStrGetSzStr(cstr_t *pThis)
if(pThis->pBuf != NULL)
if(pThis->pszBuf == NULL) {
/* we do not yet have a usable sz version - so create it... */
- if((pThis->pszBuf = malloc((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) {
+ if((pThis->pszBuf = MALLOC((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) {
/* TODO: think about what to do - so far, I have no bright
* idea... rgerhards 2005-09-07
*/
@@ -369,7 +369,7 @@ rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL)
if(pThis->pBuf == NULL) {
if(bRetNULL == 0) {
- CHKmalloc(pRetBuf = malloc(sizeof(uchar)));
+ CHKmalloc(pRetBuf = MALLOC(sizeof(uchar)));
*pRetBuf = '\0';
} else {
pRetBuf = NULL;
diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c
index 8cebf810..f448cd4f 100644
--- a/runtime/strmsrv.c
+++ b/runtime/strmsrv.c
@@ -75,6 +75,7 @@
#include "unicode-helper.h"
MODULE_TYPE_LIB
+MODULE_TYPE_NOKEEP
/* defines */
#define STRMSESS_MAX_DEFAULT 200 /* default for nbr of strm sessions if no number is given */
@@ -165,7 +166,7 @@ addNewLstnPort(strmsrv_t *pThis, uchar *pszPort)
ISOBJ_TYPE_assert(pThis, strmsrv);
/* create entry */
- CHKmalloc(pEntry = malloc(sizeof(strmLstnPortList_t)));
+ CHKmalloc(pEntry = MALLOC(sizeof(strmLstnPortList_t)));
pEntry->pszPort = pszPort;
pEntry->pSrv = pThis;
CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName));
diff --git a/runtime/sync.c b/runtime/sync.c
index a3053e28..15d5c80f 100644
--- a/runtime/sync.c
+++ b/runtime/sync.c
@@ -28,12 +28,13 @@
#include "rsyslog.h"
#include "sync.h"
+#include "debug.h"
void
SyncObjInit(pthread_mutex_t **mut)
{
- *mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t));
+ *mut = (pthread_mutex_t *) MALLOC(sizeof (pthread_mutex_t));
pthread_mutex_init(*mut, NULL);
}
diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h
index 4a26f993..fcebd985 100644
--- a/runtime/syslogd-types.h
+++ b/runtime/syslogd-types.h
@@ -56,7 +56,10 @@
* applications I do not yet envision. -- rgerhards, 2007-07-24
*/
typedef enum _syslogFeature {
- sFEATURERepeatedMsgReduction = 1
+ sFEATURERepeatedMsgReduction = 1, /* for output modules */
+ sFEATURENonCancelInputTermination = 2, /* for input modules */
+ sFEATUREAutomaticSanitazion = 3, /* for parser modules */
+ sFEATUREAutomaticPRIParsing = 4 /* for parser modules */
} syslogFeature;
/* we define our own facility and severities */
diff --git a/runtime/sysvar.c b/runtime/sysvar.c
index 4a6ace19..ecc31e2d 100644
--- a/runtime/sysvar.c
+++ b/runtime/sysvar.c
@@ -41,6 +41,7 @@
DEFobjStaticHelpers
DEFobjCurrIf(var)
DEFobjCurrIf(datetime)
+DEFobjCurrIf(glbl)
/* Standard-Constructor
@@ -146,6 +147,8 @@ GetVar(cstr_t *pstrVarName, var_t **ppVar)
CHKiRet(getNOW(NOW_HOUR, &pstrProp));
} else if(!rsCStrSzStrCmp(pstrVarName, (uchar*)"minute", sizeof("minute") - 1)) {
CHKiRet(getNOW(NOW_MINUTE, &pstrProp));
+ } else if(!rsCStrSzStrCmp(pstrVarName, (uchar*)"myhostname", sizeof("myhostname") - 1)) {
+ CHKiRet(rsCStrConstructFromszStr(&pstrProp, glbl.GetLocalHostName()));
} else {
ABORT_FINALIZE(RS_RET_SYSVAR_NOT_FOUND);
}
@@ -191,6 +194,7 @@ BEGINObjClassInit(sysvar, 1, OBJ_IS_CORE_MODULE) /* class, version */
/* request objects we use */
CHKiRet(objUse(var, CORE_COMPONENT));
CHKiRet(objUse(datetime, CORE_COMPONENT));
+ CHKiRet(objUse(glbl, CORE_COMPONENT));
/* set our own handlers */
OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, sysvarConstructFinalize);
diff --git a/runtime/typedefs.h b/runtime/typedefs.h
new file mode 100644
index 00000000..d3da7699
--- /dev/null
+++ b/runtime/typedefs.h
@@ -0,0 +1,148 @@
+/* This defines some types commonly used. Do NOT include any other
+ * rsyslog runtime file.
+ *
+ * Begun 2010-11-25 RGerhards
+ *
+ * Copyright (C) 2005-2008 by Rainer Gerhards and Adiscon GmbH
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#ifndef INCLUDED_TYPEDEFS_H
+#define INCLUDED_TYPEDEFS_H
+
+/* some universal fixed size integer defines ... */
+typedef long long int64;
+typedef long long unsigned uint64;
+typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */
+typedef char intTiny; /* 0..127! */
+typedef unsigned char uintTiny; /* 0..255! */
+
+/* define some base data types */
+
+typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */
+typedef struct aUsrp_s aUsrp_t;
+typedef struct thrdInfo thrdInfo_t;
+typedef struct obj_s obj_t;
+typedef struct ruleset_s ruleset_t;
+typedef struct rule_s rule_t;
+//typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */
+typedef struct NetAddr netAddr_t;
+typedef struct netstrms_s netstrms_t;
+typedef struct netstrm_s netstrm_t;
+typedef struct nssel_s nssel_t;
+typedef struct nspoll_s nspoll_t;
+typedef enum nsdsel_waitOp_e nsdsel_waitOp_t;
+typedef struct nsd_ptcp_s nsd_ptcp_t;
+typedef struct nsd_gtls_s nsd_gtls_t;
+typedef struct nsd_gsspi_s nsd_gsspi_t;
+typedef struct nsd_nss_s nsd_nss_t;
+typedef struct nsdsel_ptcp_s nsdsel_ptcp_t;
+typedef struct nsdsel_gtls_s nsdsel_gtls_t;
+typedef struct nsdpoll_ptcp_s nsdpoll_ptcp_t;
+typedef struct wti_s wti_t;
+typedef struct msg msg_t;
+typedef struct queue_s qqueue_t;
+typedef struct prop_s prop_t;
+typedef struct interface_s interface_t;
+typedef struct objInfo_s objInfo_t;
+typedef enum rsRetVal_ rsRetVal; /**< friendly type for global return value */
+typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function ptr to a function returning a function ptr... */
+typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */
+typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */
+typedef struct tcpsrv_s tcpsrv_t;
+typedef struct tcps_sess_s tcps_sess_t;
+typedef struct strmsrv_s strmsrv_t;
+typedef struct strms_sess_s strms_sess_t;
+typedef struct vmstk_s vmstk_t;
+typedef struct batch_obj_s batch_obj_t;
+typedef struct batch_s batch_t;
+typedef struct wtp_s wtp_t;
+typedef struct modInfo_s modInfo_t;
+typedef struct parser_s parser_t;
+typedef struct parserList_s parserList_t;
+typedef struct strgen_s strgen_t;
+typedef struct strgenList_s strgenList_t;
+typedef struct statsobj_s statsobj_t;
+typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */
+typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */
+
+typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename?
+typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename?
+
+/* under Solaris (actually only SPARC), we need to redefine some types
+ * to be void, so that we get void* pointers. Otherwise, we will see
+ * alignment errors.
+ */
+#ifdef OS_SOLARIS
+ typedef void * obj_t_ptr;
+ typedef void nsd_t;
+ typedef void nsdsel_t;
+ typedef void nsdpoll_t;
+#else
+ typedef obj_t *obj_t_ptr;
+ typedef obj_t nsd_t;
+ typedef obj_t nsdsel_t;
+ typedef obj_t nsdpoll_t;
+#endif
+
+
+#ifdef __hpux
+typedef unsigned int u_int32_t; /* TODO: is this correct? */
+typedef int socklen_t;
+#endif
+
+typedef struct epoll_event epoll_event_t;
+
+typedef char sbool; /* (small bool) I intentionally use char, to keep it slim so that many fit into the CPU cache! */
+
+/* settings for flow control
+ * TODO: is there a better place for them? -- rgerhards, 2008-03-14
+ */
+typedef enum {
+ eFLOWCTL_NO_DELAY = 0, /**< UDP and other non-delayable sources */
+ eFLOWCTL_LIGHT_DELAY = 1, /**< some light delay possible, but no extended period of time */
+ eFLOWCTL_FULL_DELAY = 2 /**< delay possible for extended period of time */
+} flowControl_t;
+
+/* filter operations */
+typedef enum {
+ FIOP_NOP = 0, /* do not use - No Operation */
+ FIOP_CONTAINS = 1, /* contains string? */
+ FIOP_ISEQUAL = 2, /* is (exactly) equal? */
+ FIOP_STARTSWITH = 3, /* starts with a string? */
+ FIOP_REGEX = 4, /* matches a (BRE) regular expression? */
+ FIOP_EREREGEX = 5 /* matches a ERE regular expression? */
+} fiop_t;
+
+
+/* multi-submit support.
+ * This is done via a simple data structure, which holds the number of elements
+ * as well as an array of to-be-submitted messages.
+ * rgerhards, 2009-06-16
+ */
+typedef struct multi_submit_s multi_submit_t;
+struct multi_submit_s {
+ short maxElem; /* maximum number of Elements */
+ short nElem; /* current number of Elements, points to the next one FREE */
+ msg_t **ppMsgs;
+};
+
+#endif /* multi-include protection */
+/* vim:set ai:
+ */
diff --git a/runtime/unlimited_select.h b/runtime/unlimited_select.h
index 32dadc03..3fa7eb06 100644
--- a/runtime/unlimited_select.h
+++ b/runtime/unlimited_select.h
@@ -23,6 +23,7 @@
*/
#ifndef UNLIMITED_SELECT_H_INCLUDED
+#define UNLIMITED_SELECT_H_INCLUDED
#include <string.h>
#include <stdlib.h>
diff --git a/runtime/vm.c b/runtime/vm.c
index 0ed174d1..84ba4bcf 100644
--- a/runtime/vm.c
+++ b/runtime/vm.c
@@ -288,7 +288,7 @@ BEGINCMPOP(CMP_NEQ) /* remember to change the name also in the END macro! */
case VARTYPE_STR:
bRes = rsCStrCStrCmp(operand1->val.pStr, operand2->val.pStr);
break;
-ENDCMPOP(CMP_EQ)
+ENDCMPOP(CMP_NEQ)
BEGINCMPOP(CMP_LT) /* remember to change the name also in the END macro! */
case VARTYPE_NUMBER:
@@ -474,6 +474,15 @@ CODESTARTop(PUSHSYSVAR)
CHKiRet(sysvar.GetVar(pOp->operand.pVar->val.pStr, &pVal));
vmstk.Push(pThis->pStk, pVal);
finalize_it:
+ if(Debug && iRet != RS_RET_OK) {
+ if(iRet == RS_RET_SYSVAR_NOT_FOUND) {
+ DBGPRINTF("rainerscript: sysvar '%s' not found\n",
+ rsCStrGetSzStrNoNULL(pOp->operand.pVar->val.pStr));
+ } else {
+ DBGPRINTF("rainerscript: error %d trying to obtain sysvar '%s'\n",
+ iRet, rsCStrGetSzStrNoNULL(pOp->operand.pVar->val.pStr));
+ }
+ }
ENDop(PUSHSYSVAR)
/* The function call operation is only very roughly implemented. While the plumbing
@@ -666,9 +675,11 @@ execProg(vm_t *pThis, vmprg_t *pProg)
ISOBJ_TYPE_assert(pThis, vm);
ISOBJ_TYPE_assert(pProg, vmprg);
-#define doOP(OP) case opcode_##OP: CHKiRet(op##OP(pThis, pCurrOp)); break
+#define doOP(OP) case opcode_##OP: DBGPRINTF("rainerscript: opcode %s\n", #OP); \
+ CHKiRet(op##OP(pThis, pCurrOp)); break
pCurrOp = pProg->vmopRoot; /* TODO: do this via a method! */
while(pCurrOp != NULL && pCurrOp->opcode != opcode_END_PROG) {
+ DBGPRINTF("rainerscript: executing step, opcode %d...\n", pCurrOp->opcode);
switch(pCurrOp->opcode) {
doOP(OR);
doOP(AND);
@@ -695,8 +706,8 @@ execProg(vm_t *pThis, vmprg_t *pProg)
doOP(UNARY_MINUS);
doOP(FUNC_CALL);
default:
- ABORT_FINALIZE(RS_RET_INVALID_VMOP);
dbgoprint((obj_t*) pThis, "invalid instruction %d in vmprg\n", pCurrOp->opcode);
+ ABORT_FINALIZE(RS_RET_INVALID_VMOP);
break;
}
/* so far, we have plain sequential execution, so on to next... */
@@ -709,6 +720,7 @@ execProg(vm_t *pThis, vmprg_t *pProg)
*/
finalize_it:
+ DBGPRINTF("rainerscript: script execution terminated with state %d\n", iRet);
RETiRet;
}
diff --git a/runtime/wti.c b/runtime/wti.c
index 90bb14ed..9343f5c5 100644
--- a/runtime/wti.c
+++ b/runtime/wti.c
@@ -39,10 +39,6 @@
#include <pthread.h>
#include <errno.h>
-#ifdef OS_SOLARIS
-# include <sched.h>
-#endif
-
#include "rsyslog.h"
#include "stringbuf.h"
#include "srUtils.h"
@@ -75,92 +71,72 @@ wtiGetDbgHdr(wti_t *pThis)
}
-/* get the current worker state. For simplicity and speed, we have
- * NOT used our regular calling interface this time. I hope that won't
- * bite in the long term... -- rgerhards, 2008-01-17
- * TODO: may be performance optimized by atomic operations
+/* return the current worker processing state. For the sake of
+ * simplicity, we do not use the iRet interface. -- rgerhards, 2009-07-17
*/
-qWrkCmd_t
-wtiGetState(wti_t *pThis, int bLockMutex)
+sbool
+wtiGetState(wti_t *pThis)
{
- DEFVARS_mutexProtection;
- qWrkCmd_t tCmd;
+ return ATOMIC_FETCH_32BIT(&pThis->bIsRunning, &pThis->mutIsRunning);
+}
- BEGINfunc
- ISOBJ_TYPE_assert(pThis, wti);
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
- tCmd = pThis->tCurrCmd;
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
+/* Set this thread to "always running" state (can not be unset)
+ * rgerhards, 2009-07-20
+ */
+rsRetVal
+wtiSetAlwaysRunning(wti_t *pThis)
+{
+ ISOBJ_TYPE_assert(pThis, wti);
+ pThis->bAlwaysRunning = TRUE;
+ return RS_RET_OK;
+}
- ENDfunc
- return tCmd;
+/* Set status (thread is running or not), actually an property of
+ * use for wtp, but we need to have it per thread instance (thus it
+ * is inside wti). -- rgerhards, 2009-07-17
+ */
+rsRetVal
+wtiSetState(wti_t *pThis, sbool bNewVal)
+{
+ ISOBJ_TYPE_assert(pThis, wti);
+ if(bNewVal) {
+ ATOMIC_STORE_1_TO_INT(&pThis->bIsRunning, &pThis->mutIsRunning);
+ } else {
+ ATOMIC_STORE_0_TO_INT(&pThis->bIsRunning, &pThis->mutIsRunning);
+ }
+ return RS_RET_OK;
}
-/* send a command to a specific thread
- * bActiveOnly specifies if the command should be sent only when the worker is
- * in an active state. -- rgerhards, 2008-01-20
+/* advise all workers to start by interrupting them. That should unblock all srSleep()
+ * calls.
*/
rsRetVal
-wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex)
+wtiWakeupThrd(wti_t *pThis)
{
DEFiRet;
- qWrkCmd_t tCurrCmd;
- DEFVARS_mutexProtection;
ISOBJ_TYPE_assert(pThis, wti);
- assert(tCmd <= eWRKTHRD_SHUTDOWN_IMMEDIATE);
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
-
- tCurrCmd = pThis->tCurrCmd;
- /* all worker states must be followed sequentially, only termination can be set in any state */
- if( (bActiveOnly && (tCurrCmd < eWRKTHRD_RUN_CREATED))
- || (tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) {
- DBGPRINTF("%s: command %d can not be accepted in current %d processing state - ignored\n",
- wtiGetDbgHdr(pThis), tCmd, tCurrCmd);
- } else {
- DBGPRINTF("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd);
- /* we could replace this with a simple if, but we leave the switch in in case we need
- * to add something at a later stage. -- rgerhards, 2008-09-30
- */
- switch(tCmd) {
- case eWRKTHRD_TERMINATING:
- /* TODO: re-enable meaningful debug msg! (via function callback?)
- dbgprintf("%s: thread terminating with %d entries left in queue, %d workers running.\n",
- wtiGetDbgHdr(pThis->pQueue), pThis->pQueue->iQueueSize,
- pThis->pQueue->iCurNumWrkThrd);
- */
- pthread_cond_signal(&pThis->condExitDone);
- dbgprintf("%s: worker terminating\n", wtiGetDbgHdr(pThis));
- break;
- /* these cases just to satisfy the compiler, we do (yet) not act an them: */
- case eWRKTHRD_RUNNING:
- case eWRKTHRD_STOPPED:
- case eWRKTHRD_RUN_CREATED:
- case eWRKTHRD_RUN_INIT:
- case eWRKTHRD_SHUTDOWN:
- case eWRKTHRD_SHUTDOWN_IMMEDIATE:
- /* DO NOTHING */
- break;
- }
- /* apply the new state */
- unsigned val = ATOMIC_CAS_VAL((int*)&pThis->tCurrCmd, tCurrCmd, tCmd, &pThis->mutCurrCmd);
- if(val != tCurrCmd) {
- DBGPRINTF("wtiSetState PROBLEM, tCurrCmd %d overwritten with %d, wanted to set %d\n", tCurrCmd, val, tCmd);
- }
+ if(wtiGetState(pThis)) {
+ /* we first try the cooperative "cancel" interface */
+ pthread_kill(pThis->thrdID, SIGTTIN);
+ dbgprintf("sent SIGTTIN to worker thread 0x%x\n", (unsigned) pThis->thrdID);
}
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
RETiRet;
}
-/* Cancel the thread. If the thread is already cancelled or terminated,
- * we do not again cancel it. But it is save and legal to call wtiCancelThrd() in
- * such situations.
+/* Cancel the thread. If the thread is not running. But it is save and legal to
+ * call wtiCancelThrd() in such situations. This function only returns when the
+ * thread has terminated. Else we may get race conditions all over the code...
+ * Note that when waiting for the thread to terminate, we do a busy wait, checking
+ * progress every 10ms. It is very unlikely that we will ever cancel a thread
+ * and, if so, it will only happen at the end of the rsyslog run. So doing this
+ * kind of non-optimal wait is considered preferable over using condition variables.
* rgerhards, 2008-02-26
*/
rsRetVal
@@ -170,19 +146,24 @@ wtiCancelThrd(wti_t *pThis)
ISOBJ_TYPE_assert(pThis, wti);
- d_pthread_mutex_lock(&pThis->mut);
- wtiProcessThrdChanges(pThis, MUTEX_ALREADY_LOCKED); /* process state change, so that we have current state vars */
+ if(wtiGetState(pThis)) {
+ /* we first try the cooperative "cancel" interface */
+ pthread_kill(pThis->thrdID, SIGTTIN);
+ dbgprintf("sent SIGTTIN to worker thread 0x%x, giving it a chance to terminate\n", (unsigned) pThis->thrdID);
+ srSleep(0, 10000);
+ }
- if(pThis->tCurrCmd >= eWRKTHRD_TERMINATING) {
- dbgoprint((obj_t*) pThis, "canceling worker thread, curr stat %d\n", pThis->tCurrCmd);
+ if(wtiGetState(pThis)) {
+ dbgprintf("cooperative worker termination failed, using cancellation...\n");
+ dbgoprint((obj_t*) pThis, "canceling worker thread\n");
pthread_cancel(pThis->thrdID);
- wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED);
- wtpSetThrdStateChanged(pThis->pWtp, 1); /* indicate change, so harverster will be called */
+ /* now wait until the thread terminates... */
+ while(wtiGetState(pThis)) {
+ srSleep(0, 10000);
+ }
}
- d_pthread_mutex_unlock(&pThis->mut);
-
RETiRet;
}
@@ -190,26 +171,9 @@ wtiCancelThrd(wti_t *pThis)
/* Destructor */
BEGINobjDestruct(wti) /* be sure to specify the object type also in END and CODESTART macros! */
CODESTARTobjDestruct(wti)
- /* if we reach this point, we must make sure the associated worker has terminated. It is
- * the callers duty to make sure the worker already knows it shall terminate.
- * TODO: is it *really* the caller's duty? ...mmmhhhh.... smells bad... rgerhards, 2008-01-25
- */
- wtiProcessThrdChanges(pThis, LOCK_MUTEX); /* process state change one last time */
-
- d_pthread_mutex_lock(&pThis->mut);
- if(wtiGetState(pThis, MUTEX_ALREADY_LOCKED) != eWRKTHRD_STOPPED) {
- dbgprintf("%s: WARNING: worker %p shall be destructed but is still running (might be OK) - joining it\n",
- wtiGetDbgHdr(pThis), pThis);
- /* let's hope the caller actually instructed it to shutdown... */
- pthread_cond_wait(&pThis->condExitDone, &pThis->mut);
- wtiJoinThrd(pThis);
- }
- d_pthread_mutex_unlock(&pThis->mut);
-
/* actual destruction */
- pthread_cond_destroy(&pThis->condExitDone);
- pthread_mutex_destroy(&pThis->mut);
- DESTROY_ATOMIC_HELPER_MUT(pThis->mutCurrCmd);
+ batchFree(&pThis->batch);
+ DESTROY_ATOMIC_HELPER_MUT(pThis->mutIsRunning);
free(pThis->pszDbgHdr);
ENDobjDestruct(wti)
@@ -218,9 +182,7 @@ ENDobjDestruct(wti)
/* Standard-Constructor for the wti object
*/
BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */
- pthread_cond_init(&pThis->condExitDone, NULL);
- pthread_mutex_init(&pThis->mut, NULL);
- INIT_ATOMIC_HELPER_MUT(pThis->mutCurrCmd);
+ INIT_ATOMIC_HELPER_MUT(pThis->mutIsRunning);
ENDobjConstruct(wti)
@@ -231,81 +193,28 @@ rsRetVal
wtiConstructFinalize(wti_t *pThis)
{
DEFiRet;
+ int iDeqBatchSize;
ISOBJ_TYPE_assert(pThis, wti);
dbgprintf("%s: finalizing construction of worker instance data\n", wtiGetDbgHdr(pThis));
- /* initialize our thread instance descriptor */
- pThis->pUsrp = NULL;
- pThis->tCurrCmd = eWRKTHRD_STOPPED;
-
- RETiRet;
-}
-
-
-/* join a specific worker thread
- * we do not lock the mutex, because join will sync anyways...
- */
-rsRetVal
-wtiJoinThrd(wti_t *pThis)
-{
- DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, wti);
- dbgprintf("waiting for worker %s termination, current state %d\n", wtiGetDbgHdr(pThis), pThis->tCurrCmd);
- if (pThis->thrdID == 0) {
- dbgprintf("worker %s was already stopped\n", wtiGetDbgHdr(pThis));
- } else {
- pthread_join(pThis->thrdID, NULL);
- wtiSetState(pThis, eWRKTHRD_STOPPED, 0, MUTEX_ALREADY_LOCKED); /* back to virgin... */
- pThis->thrdID = 0; /* invalidate the thread ID so that we do not accidently find reused ones */
- dbgprintf("worker %s has stopped\n", wtiGetDbgHdr(pThis));
- }
-
- RETiRet;
-}
-
-/* check if we had a worker thread changes and, if so, act
- * on it. At a minimum, terminated threads are harvested (joined).
- */
-rsRetVal
-wtiProcessThrdChanges(wti_t *pThis, int bLockMutex)
-{
- DEFiRet;
- DEFVARS_mutexProtection;
+ /* initialize our thread instance descriptor (no concurrency here) */
+ pThis->bIsRunning = FALSE;
- ISOBJ_TYPE_assert(pThis, wti);
-
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
- switch(pThis->tCurrCmd) {
- case eWRKTHRD_TERMINATING:
- /* we need to at least temporarily release the mutex, because otherwise
- * we may deadlock with the thread we intend to join (it aquires the mutex
- * during termination processing). -- rgerhards, 2008-02-26
- */
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
- iRet = wtiJoinThrd(pThis);
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
- break;
- /* these cases just to satisfy the compiler, we do not act an them: */
- case eWRKTHRD_STOPPED:
- case eWRKTHRD_RUN_CREATED:
- case eWRKTHRD_RUN_INIT:
- case eWRKTHRD_RUNNING:
- case eWRKTHRD_SHUTDOWN:
- case eWRKTHRD_SHUTDOWN_IMMEDIATE:
- /* DO NOTHING */
- break;
- }
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
+ /* we now alloc the array for user pointers. We obtain the max from the queue itself. */
+ CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize));
+ CHKiRet(batchInit(&pThis->batch, iDeqBatchSize));
+finalize_it:
RETiRet;
}
/* cancellation cleanup handler for queueWorker ()
- * Updates admin structure and frees ressources.
+ * Most importantly, it must bring back the batch into a consistent state.
+ * Keep in mind that cancellation is disabled if we run into
+ * the cancel cleanup handler (and have been cancelled).
* rgerhards, 2008-01-16
*/
static void
@@ -313,7 +222,6 @@ wtiWorkerCancelCleanup(void *arg)
{
wti_t *pThis = (wti_t*) arg;
wtp_t *pWtp;
- int iCancelStateSave;
BEGINfunc
ISOBJ_TYPE_assert(pThis, wti);
@@ -321,102 +229,108 @@ wtiWorkerCancelCleanup(void *arg)
ISOBJ_TYPE_assert(pWtp, wtp);
DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis));
+ pWtp->pfObjProcessed(pWtp->pUsr, pThis);
+ DBGPRINTF("%s: done cancelation cleanup handler.\n", wtiGetDbgHdr(pThis));
- /* call user supplied handler (that one e.g. requeues the element) */
- pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp);
+ ENDfunc
+}
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
- d_pthread_mutex_lock(&pWtp->mut);
- wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED);
- wtpSetThrdStateChanged(pWtp, 1); /* indicate change, so harverster will be called */
- d_pthread_mutex_unlock(&pWtp->mut);
- pthread_setcancelstate(iCancelStateSave, NULL);
+/* wait for queue to become non-empty or timeout
+ * helper to wtiWorker. Note the the predicate is
+ * re-tested by the caller, so it is OK to NOT do it here.
+ * rgerhards, 2009-05-20
+ */
+static inline void
+doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured)
+{
+ struct timespec t;
+
+ BEGINfunc
+ DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis));
+
+ if(pThis->bAlwaysRunning) {
+ /* never shut down any started worker */
+ d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr);
+ } else {
+ timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */
+ if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) {
+ DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis));
+ *pbInactivityTOOccured = 1; /* indicate we had a timeout */
+ }
+ }
+ dbgoprint((obj_t*) pThis, "worker awoke from idle processing\n");
ENDfunc
}
-/* generic worker thread framework
+/* generic worker thread framework. Note that we prohibit cancellation
+ * during almost all times, because it can have very undesired side effects.
+ * However, we may need to cancel a thread if the consumer blocks for too
+ * long (during shutdown). So what we do is block cancellation, and every
+ * consumer must enable it during the periods where it is safe.
*/
#pragma GCC diagnostic ignored "-Wempty-body"
rsRetVal
wtiWorker(wti_t *pThis)
{
- DEFiRet;
- DEFVARS_mutexProtection;
- struct timespec t;
wtp_t *pWtp; /* our worker thread pool */
int bInactivityTOOccured = 0;
+ rsRetVal localRet;
+ rsRetVal terminateRet;
+ int iCancelStateSave;
+ DEFiRet;
ISOBJ_TYPE_assert(pThis, wti);
pWtp = pThis->pWtp; /* shortcut */
ISOBJ_TYPE_assert(pWtp, wtp);
dbgSetThrdName(pThis->pszDbgHdr);
- pThis->pUsrp = NULL;
pthread_cleanup_push(wtiWorkerCancelCleanup, pThis);
-
- BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX);
- pWtp->pfOnWorkerStartup(pWtp->pUsr);
- END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr);
+ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
/* now we have our identity, on to real processing */
while(1) { /* loop will be broken below - need to do mutex locks */
- /* process any pending thread requests */
- wtpProcessThrdChanges(pWtp);
-
- /* if we have a rate-limiter set for this worker pool, let's call it. Please
- * keep in mind that the rate-limiter may hold us for an extended period
- * of time. -- rgerhards, 2008-04-02
- */
- if(pWtp->pfRateLimiter != NULL) {
+ if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */
pWtp->pfRateLimiter(pWtp->pUsr);
}
- wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */
- BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX);
-
- if( (bInactivityTOOccured && pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED))
- || wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED)) {
- END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr);
- break; /* end worker thread run */
+ d_pthread_mutex_lock(pWtp->pmutUsr);
+
+ /* first check if we are in shutdown process (but evaluate a bit later) */
+ terminateRet = wtpChkStopWrkr(pWtp, MUTEX_ALREADY_LOCKED);
+ if(terminateRet == RS_RET_TERMINATE_NOW) {
+ /* we now need to free the old batch */
+ localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis);
+ dbgoprint((obj_t*) pThis, "terminating worker because of TERMINATE_NOW mode, del iRet %d\n",
+ localRet);
+ d_pthread_mutex_unlock(pWtp->pmutUsr);
+ break;
}
- bInactivityTOOccured = 0; /* reset for next run */
- /* if we reach this point, we are still protected by the mutex */
-
- if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) {
- DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis));
- pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED);
-
- if(pWtp->toWrkShutdown == -1) {
- /* never shut down any started worker */
- d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr);
- } else {
- timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */
- if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) {
- DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis));
- bInactivityTOOccured = 1; /* indicate we had a timeout */
- }
+ /* try to execute and process whatever we have */
+ /* Note that this function releases and re-aquires the mutex. The returned
+ * information on idle state must be processed before releasing the mutex again.
+ */
+ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis);
+
+ if(localRet == RS_RET_IDLE) {
+ if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) {
+ d_pthread_mutex_unlock(pWtp->pmutUsr);
+ break; /* end of loop */
}
- END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr);
+ doIdleProcessing(pThis, pWtp, &bInactivityTOOccured);
+ d_pthread_mutex_unlock(pWtp->pmutUsr);
continue; /* request next iteration */
}
- /* if we reach this point, we have a non-empty queue (and are still protected by mutex) */
- pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave);
+ d_pthread_mutex_unlock(pWtp->pmutUsr);
+
+ bInactivityTOOccured = 0; /* reset for next run */
}
/* indicate termination */
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
- d_pthread_mutex_lock(&pThis->mut);
pthread_cleanup_pop(0); /* remove cleanup handler */
-
- pWtp->pfOnWorkerShutdown(pWtp->pUsr);
-
- wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED);
- wtpSetThrdStateChanged(pWtp, 1); /* indicate change, so harverster will be called */
- d_pthread_mutex_unlock(&pThis->mut);
pthread_setcancelstate(iCancelStateSave, NULL);
RETiRet;
@@ -445,10 +359,9 @@ wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg)
if(pThis->pszDbgHdr != NULL) {
free(pThis->pszDbgHdr);
- pThis->pszDbgHdr = NULL;
}
- if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL)
+ if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL)
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */
@@ -479,6 +392,5 @@ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most
CHKiRet(objUse(glbl, CORE_COMPONENT));
ENDObjClassInit(wti)
-/*
- * vi:set ai:
+/* vi:set ai:
*/
diff --git a/runtime/wti.h b/runtime/wti.h
index d81672f3..51ece4ef 100644
--- a/runtime/wti.h
+++ b/runtime/wti.h
@@ -1,6 +1,6 @@
/* Definition of the worker thread instance (wti) class.
*
- * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2008, 2009 by Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -27,22 +27,20 @@
#include <pthread.h>
#include "wtp.h"
#include "obj.h"
+#include "batch.h"
+
/* the worker thread instance class */
-typedef struct wti_s {
+struct wti_s {
BEGINobjInstance;
- pthread_t thrdID; /* thread ID */
- qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */
- obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */
+ pthread_t thrdID; /* thread ID */
+ int bIsRunning; /* is this thread currently running? (must be int for atomic op!) */
+ sbool bAlwaysRunning; /* should this thread always run? */
wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */
- pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */
- pthread_mutex_t mut;
- bool bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */
+ batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */
uchar *pszDbgHdr; /* header string for debug messages */
- DEF_ATOMIC_HELPER_MUT(mutCurrCmd);
-} wti_t;
-
-/* some symbolic constants for easier reference */
+ DEF_ATOMIC_HELPER_MUT(mutIsRunning);
+};
/* prototypes */
@@ -50,12 +48,12 @@ rsRetVal wtiConstruct(wti_t **ppThis);
rsRetVal wtiConstructFinalize(wti_t *pThis);
rsRetVal wtiDestruct(wti_t **ppThis);
rsRetVal wtiWorker(wti_t *pThis);
-rsRetVal wtiProcessThrdChanges(wti_t *pThis, int bLockMutex);
rsRetVal wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg);
-rsRetVal wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex);
-rsRetVal wtiJoinThrd(wti_t *pThis);
rsRetVal wtiCancelThrd(wti_t *pThis);
-qWrkCmd_t wtiGetState(wti_t *pThis, int bLockMutex);
+rsRetVal wtiSetAlwaysRunning(wti_t *pThis);
+rsRetVal wtiSetState(wti_t *pThis, sbool bNew);
+rsRetVal wtiWakeupThrd(wti_t *pThis);
+sbool wtiGetState(wti_t *pThis);
PROTOTYPEObjClassInit(wti);
PROTOTYPEpropSetMeth(wti, pszDbgHdr, uchar*);
PROTOTYPEpropSetMeth(wti, pWtp, wtp_t*);
diff --git a/runtime/wtp.c b/runtime/wtp.c
index b4fd2e04..e615fb19 100644
--- a/runtime/wtp.c
+++ b/runtime/wtp.c
@@ -8,7 +8,7 @@
* (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it
* if you are getting aquainted to the object.
*
- * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
@@ -44,9 +44,10 @@
# include <sys/prctl.h>
#endif
-#ifdef OS_SOLARIS
-# include <sched.h>
-#endif
+/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20
+//#ifdef OS_SOLARIS
+//# include <sched.h>
+//#endif
#include "rsyslog.h"
#include "stringbuf.h"
@@ -82,22 +83,27 @@ wtpGetDbgHdr(wtp_t *pThis)
/* Not implemented dummy function for constructor */
-static rsRetVal NotImplementedDummy() { return RS_RET_OK; }
+static rsRetVal NotImplementedDummy() { return RS_RET_NOT_IMPLEMENTED; }
/* Standard-Constructor for the wtp object
*/
BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */
- pthread_mutex_init(&pThis->mut, NULL);
- pthread_mutex_init(&pThis->mutThrdShutdwn, NULL);
+ pthread_mutex_init(&pThis->mutWtp, NULL);
pthread_cond_init(&pThis->condThrdTrm, NULL);
+ pthread_attr_init(&pThis->attrThrd);
+ /* Set thread scheduling policy to default */
+#ifdef HAVE_PTHREAD_SETSCHEDPARAM
+ pthread_attr_setschedpolicy(&pThis->attrThrd, default_thr_sched_policy);
+ pthread_attr_setschedparam(&pThis->attrThrd, &default_sched_param);
+ pthread_attr_setinheritsched(&pThis->attrThrd, PTHREAD_EXPLICIT_SCHED);
+#endif
+ pthread_attr_setdetachstate(&pThis->attrThrd, PTHREAD_CREATE_DETACHED);
/* set all function pointers to "not implemented" dummy so that we can safely call them */
pThis->pfChkStopWrkr = NotImplementedDummy;
- pThis->pfIsIdle = NotImplementedDummy;
+ pThis->pfGetDeqBatchSize = NotImplementedDummy;
pThis->pfDoWork = NotImplementedDummy;
- pThis->pfOnIdle = NotImplementedDummy;
- pThis->pfOnWorkerCancel = NotImplementedDummy;
- pThis->pfOnWorkerStartup = NotImplementedDummy;
- pThis->pfOnWorkerShutdown = NotImplementedDummy;
- INIT_ATOMIC_HELPER_MUT(pThis->mutThrdStateChanged);
+ pThis->pfObjProcessed = NotImplementedDummy;
+ INIT_ATOMIC_HELPER_MUT(pThis->mutCurNumWrkThrd);
+ INIT_ATOMIC_HELPER_MUT(pThis->mutWtpState);
ENDobjConstruct(wtp)
@@ -115,13 +121,12 @@ wtpConstructFinalize(wtp_t *pThis)
ISOBJ_TYPE_assert(pThis, wtp);
- dbgprintf("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis));
+ DBGPRINTF("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis));
/* alloc and construct workers - this can only be done in finalizer as we previously do
* not know the max number of workers
*/
- if((pThis->pWrkr = malloc(sizeof(wti_t*) * pThis->iNumWorkerThreads)) == NULL)
- ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
-
+ CHKmalloc(pThis->pWrkr = MALLOC(sizeof(wti_t*) * pThis->iNumWorkerThreads));
+
for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) {
CHKiRet(wtiConstruct(&pThis->pWrkr[i]));
pWti = pThis->pWrkr[i];
@@ -141,8 +146,6 @@ finalize_it:
BEGINobjDestruct(wtp) /* be sure to specify the object type also in END and CODESTART macros! */
int i;
CODESTARTobjDestruct(wtp)
- wtpProcessThrdChanges(pThis); /* process thread changes one last time */
-
/* destruct workers */
for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i)
wtiDestruct(&pThis->pWrkr[i]);
@@ -152,149 +155,70 @@ CODESTARTobjDestruct(wtp)
/* actual destruction */
pthread_cond_destroy(&pThis->condThrdTrm);
- pthread_mutex_destroy(&pThis->mut);
- pthread_mutex_destroy(&pThis->mutThrdShutdwn);
- DESTROY_ATOMIC_HELPER_MUT(pThis->mutThrdStateChanged);
+ pthread_mutex_destroy(&pThis->mutWtp);
+ pthread_attr_destroy(&pThis->attrThrd);
+ DESTROY_ATOMIC_HELPER_MUT(pThis->mutCurNumWrkThrd);
+ DESTROY_ATOMIC_HELPER_MUT(pThis->mutWtpState);
free(pThis->pszDbgHdr);
ENDobjDestruct(wtp)
-/* wake up at least one worker thread.
- * rgerhards, 2008-01-20
- */
-rsRetVal
-wtpWakeupWrkr(wtp_t *pThis)
-{
- DEFiRet;
-
- /* TODO; mutex? I think not needed, as we do not need predictable exec order -- rgerhards, 2008-01-28 */
- ISOBJ_TYPE_assert(pThis, wtp);
- pthread_cond_signal(pThis->pcondBusy);
- RETiRet;
-}
-
-/* wake up all worker threads.
- * rgerhards, 2008-01-16
- */
-rsRetVal
-wtpWakeupAllWrkr(wtp_t *pThis)
-{
- DEFiRet;
-
- ISOBJ_TYPE_assert(pThis, wtp);
- pthread_cond_broadcast(pThis->pcondBusy);
- RETiRet;
-}
-
-
-/* set the bThrdStateChanged in an atomic way. Note that
- * val may only be 0 or 1.
- */
-void
-wtpSetThrdStateChanged(wtp_t *pThis, int val)
-{
- if(val == 0) {
- ATOMIC_STORE_0_TO_INT(&pThis->bThrdStateChanged, pThis->mutThrdStateChanged);
- } else {
- ATOMIC_STORE_1_TO_INT(&pThis->bThrdStateChanged, pThis->mutThrdStateChanged);
- }
-}
-
-
-/* check if we had any worker thread changes and, if so, act
- * on them. At a minimum, terminated threads are harvested (joined).
- * This function MUST NEVER block on the queue mutex!
- */
-rsRetVal
-wtpProcessThrdChanges(wtp_t *pThis)
-{
- DEFiRet;
- int i;
-
- ISOBJ_TYPE_assert(pThis, wtp);
-
- if(pThis->bThrdStateChanged == 0)
- FINALIZE;
-
- if(d_pthread_mutex_trylock(&(pThis->mutThrdShutdwn)) != 0) {
- /* another thread is already in the loop */
- FINALIZE;
- }
-
- /* Note: there is a left-over potential race condition below:
- * pThis->bThrdStateChanged may be re-set by another thread while
- * we work on it and thus the loop may terminate too early. However,
- * there are no really bad effects from that so I perfer - for this
- * version - to live with the problem as is. Not a good idea to
- * introduce that large change into the stable branch without very
- * good reason. -- rgerhards, 2009-04-02
- */
- do {
- /* reset the change marker */
- wtpSetThrdStateChanged(pThis, 0);
- /* go through all threads */
- for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) {
- wtiProcessThrdChanges(pThis->pWrkr[i], LOCK_MUTEX);
- }
- /* restart if another change occured while we were processing the changes */
- } while(pThis->bThrdStateChanged != 0);
-
- d_pthread_mutex_unlock(&(pThis->mutThrdShutdwn));
-
-finalize_it:
- RETiRet;
-}
-
-
-/* Sent a specific state for the worker thread pool.
- * rgerhards, 2008-01-21
+/* Sent a specific state for the worker thread pool. -- rgerhards, 2008-01-21
+ * We do not need to do atomic instructions as set operations are only
+ * called when terminating the pool, and then in strict sequence. So we
+ * can never overwrite each other. On the other hand, it also doesn't
+ * matter if the read operation obtains an older value, as we then simply
+ * do one more iteration, what is perfectly legal (during shutdown
+ * they are awoken in any case). -- rgerhards, 2009-07-20
*/
rsRetVal
wtpSetState(wtp_t *pThis, wtpState_t iNewState)
{
- DEFiRet;
-
ISOBJ_TYPE_assert(pThis, wtp);
- pThis->wtpState = iNewState;
- /* TODO: must wakeup workers? seen to be not needed -- rgerhards, 2008-01-28 */
-
- RETiRet;
+ pThis->wtpState = iNewState; // TODO: do we need a mutex here? 2010-04-26
+ return RS_RET_OK;
}
/* check if the worker shall shutdown (1 = yes, 0 = no)
- * TODO: check if we can use atomic operations to enhance performance
* Note: there may be two mutexes locked, the bLockUsrMutex is the one in our "user"
* (e.g. the queue clas)
* rgerhards, 2008-01-21
*/
rsRetVal
-wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex)
+wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex)
{
DEFiRet;
- DEFVARS_mutexProtection;
+ wtpState_t wtpState;
ISOBJ_TYPE_assert(pThis, wtp);
+ /* we need a consistent value, but it doesn't really matter if it is changed
+ * right after the fetch - then we simply do one more iteration in the worker
+ */
+ wtpState = (wtpState_t) ATOMIC_FETCH_32BIT((int*)&pThis->wtpState, &pThis->mutWtpState);
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
- if( (pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE)
- || ((pThis->wtpState == wtpState_SHUTDOWN) && pThis->pfIsIdle(pThis->pUsr, bLockUsrMutex)))
- iRet = RS_RET_TERMINATE_NOW;
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
+ if(wtpState == wtpState_SHUTDOWN_IMMEDIATE) {
+ ABORT_FINALIZE(RS_RET_TERMINATE_NOW);
+ } else if(wtpState == wtpState_SHUTDOWN) {
+ ABORT_FINALIZE(RS_RET_TERMINATE_WHEN_IDLE);
+ }
/* try customer handler if one was set and we do not yet have a definite result */
- if(iRet == RS_RET_OK && pThis->pfChkStopWrkr != NULL) {
+ if(pThis->pfChkStopWrkr != NULL) {
iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex);
}
+finalize_it:
RETiRet;
}
#pragma GCC diagnostic ignored "-Wempty-body"
/* Send a shutdown command to all workers and see if they terminate.
- * A timeout may be specified.
+ * A timeout may be specified. This function may also be called with
+ * the current number of workers being 0, in which case it does not
+ * shut down any worker.
* rgerhards, 2008-01-14
*/
rsRetVal
@@ -302,72 +226,50 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout
{
DEFiRet;
int bTimedOut;
- int iCancelStateSave;
+ int i;
ISOBJ_TYPE_assert(pThis, wtp);
+ /* lock mutex to prevent races (may otherwise happen during idle processing and such...) */
+ d_pthread_mutex_lock(pThis->pmutUsr);
wtpSetState(pThis, tShutdownCmd);
- wtpWakeupAllWrkr(pThis);
+ pthread_cond_broadcast(pThis->pcondBusy); /* wake up all workers */
+ /* awake workers in retry loop */
+ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) {
+ wtiWakeupThrd(pThis->pWrkr[i]);
+ }
+ d_pthread_mutex_unlock(pThis->pmutUsr);
- /* see if we need to harvest (join) any terminated threads (even in timeout case,
- * some may have terminated...
- */
- wtpProcessThrdChanges(pThis);
-
- /* and wait for their termination */
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
- d_pthread_mutex_lock(&pThis->mut);
- pthread_cleanup_push(mutexCancelCleanup, &pThis->mut);
- pthread_setcancelstate(iCancelStateSave, NULL);
+ /* wait for worker thread termination */
+ d_pthread_mutex_lock(&pThis->mutWtp);
+ pthread_cleanup_push(mutexCancelCleanup, &pThis->mutWtp);
bTimedOut = 0;
while(pThis->iCurNumWrkThrd > 0 && !bTimedOut) {
- dbgprintf("%s: waiting %ldms on worker thread termination, %d still running\n",
- wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), pThis->iCurNumWrkThrd);
+ DBGPRINTF("%s: waiting %ldms on worker thread termination, %d still running\n",
+ wtpGetDbgHdr(pThis), timeoutVal(ptTimeout),
+ ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd));
- if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mut, ptTimeout) != 0) {
- dbgprintf("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis));
+ if(d_pthread_cond_timedwait(&pThis->condThrdTrm, &pThis->mutWtp, ptTimeout) != 0) {
+ DBGPRINTF("%s: timeout waiting on worker thread termination\n", wtpGetDbgHdr(pThis));
bTimedOut = 1; /* we exit the loop on timeout */
}
+
+ /* awake workers in retry loop */
+ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) {
+ wtiWakeupThrd(pThis->pWrkr[i]);
+ }
+
}
pthread_cleanup_pop(1);
if(bTimedOut)
iRet = RS_RET_TIMED_OUT;
- /* see if we need to harvest (join) any terminated threads (even in timeout case,
- * some may have terminated...
- */
- wtpProcessThrdChanges(pThis);
-
RETiRet;
}
#pragma GCC diagnostic warning "-Wempty-body"
-/* indicate that a thread has terminated and awake anyone waiting on it
- * rgerhards, 2008-01-23
- */
-rsRetVal wtpSignalWrkrTermination(wtp_t *pThis)
-{
- DEFiRet;
- /* I leave the mutex code here out as it gives us deadlocks. I think it is not really
- * needed and we are on the safe side. I leave this comment in if practice proves us
- * wrong. The whole thing should be removed after half a year or year if we see there
- * actually is no issue (or revisit it from a theoretical POV).
- * rgerhards, 2008-01-28
- * revisited 2008-09-30, still a bit unclear, leave in
- */
- /*TODO: mutex or not mutex, that's the question ;)DEFVARS_mutexProtection;*/
-
- ISOBJ_TYPE_assert(pThis, wtp);
-
- /*BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX);*/
- pthread_cond_signal(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */
- /*END_MTX_PROTECTED_OPERATIONS(&pThis->mut);*/
- RETiRet;
-}
-
-
/* Unconditionally cancel all running worker threads.
* rgerhards, 2008-01-14
*/
@@ -379,12 +281,8 @@ wtpCancelAll(wtp_t *pThis)
ISOBJ_TYPE_assert(pThis, wtp);
- /* process any pending thread requests so that we know who actually is still running */
- wtpProcessThrdChanges(pThis);
-
/* go through all workers and cancel those that are active */
for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) {
- dbgprintf("%s: try canceling worker thread %d\n", wtpGetDbgHdr(pThis), i);
wtiCancelThrd(pThis->pWrkr[i]);
}
@@ -392,40 +290,57 @@ wtpCancelAll(wtp_t *pThis)
}
-
-/* Set the Inactivity Guard
- * rgerhards, 2008-01-21
+/* this function contains shared code for both regular worker shutdown as
+ * well as shutdown via cancellation. We can not simply use pthread_cleanup_pop(1)
+ * as this introduces a race in the debug system (RETiRet system).
+ * rgerhards, 2009-10-26
*/
-rsRetVal
-wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex)
+static inline void
+wtpWrkrExecCleanup(wti_t *pWti)
{
- DEFiRet;
- DEFVARS_mutexProtection;
+ wtp_t *pThis;
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
- pThis->bInactivityGuard = bNewState;
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
+ BEGINfunc
+ ISOBJ_TYPE_assert(pWti, wti);
+ pThis = pWti->pWtp;
+ ISOBJ_TYPE_assert(pThis, wtp);
- RETiRet;
+ /* the order of the next two statements is important! */
+ wtiSetState(pWti, WRKTHRD_STOPPED);
+ ATOMIC_DEC(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd);
+
+ DBGPRINTF("%s: Worker thread %lx, terminated, um workers now %d\n",
+ wtpGetDbgHdr(pThis), (unsigned long) pWti,
+ ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd));
+
+ ENDfunc
}
-/* cancellation cleanup handler for executing worker
- * decrements the worker counter
- * rgerhards, 2008-01-20
+/* cancellation cleanup handler for executing worker decrements the worker counter.
+ * rgerhards, 2009-07-20
*/
-void
+static void
wtpWrkrExecCancelCleanup(void *arg)
{
- wtp_t *pThis = (wtp_t*) arg;
+ wti_t *pWti = (wti_t*) arg;
+ wtp_t *pThis;
BEGINfunc
+ ISOBJ_TYPE_assert(pWti, wti);
+ pThis = pWti->pWtp;
ISOBJ_TYPE_assert(pThis, wtp);
- pThis->iCurNumWrkThrd--;
- wtpSignalWrkrTermination(pThis);
+ DBGPRINTF("%s: Worker thread %lx requested to be cancelled.\n",
+ wtpGetDbgHdr(pThis), (unsigned long) pWti);
+
+ wtpWrkrExecCleanup(pWti);
- dbgprintf("%s: thread CANCELED with %d workers running.\n", wtpGetDbgHdr(pThis), pThis->iCurNumWrkThrd);
ENDfunc
+ /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main
+ * thread after the broadcast, which could destroy the debug class, resulting in a potential
+ * segfault. So we need to do the broadcast as actually the last action in our processing
+ */
+ pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */
}
@@ -437,8 +352,6 @@ wtpWrkrExecCancelCleanup(void *arg)
static void *
wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in wtp! */
{
- DEFiRet;
- DEFVARS_mutexProtection;
wti_t *pWti = (wti_t*) arg;
wtp_t *pThis;
sigset_t sigSet;
@@ -447,13 +360,20 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in
uchar thrdName[32] = "rs:";
# endif
+ BEGINfunc
ISOBJ_TYPE_assert(pWti, wti);
pThis = pWti->pWtp;
ISOBJ_TYPE_assert(pThis, wtp);
+ /* block all signals */
sigfillset(&sigSet);
pthread_sigmask(SIG_BLOCK, &sigSet, NULL);
+ /* but ignore SIGTTN, which we (ab)use to signal the thread to shutdown -- rgerhards, 2009-07-20 */
+ sigemptyset(&sigSet);
+ sigaddset(&sigSet, SIGTTIN);
+ pthread_sigmask(SIG_UNBLOCK, &sigSet, NULL);
+
# if HAVE_PRCTL && defined PR_SET_NAME
/* set thread name - we ignore if the call fails, has no harsh consequences... */
pszDbgHdr = wtpGetDbgHdr(pThis);
@@ -463,41 +383,17 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in
}
# endif
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX);
-
- /* do some late initialization */
-
- pthread_cleanup_push(wtpWrkrExecCancelCleanup, pThis);
-
- /* finally change to RUNNING state. We need to check if we actually should still run,
- * because someone may have requested us to shut down even before we got a chance to do
- * our init. That would be a bad race... -- rgerhards, 2008-01-16
- */
- wtiSetState(pWti, eWRKTHRD_RUNNING, 0, MUTEX_ALREADY_LOCKED); /* we are running now! */
-
- do {
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
-
- wtiWorker(pWti); /* just to make sure: this is NOT protected by the mutex! */
-
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX);
- } while(pThis->iCurNumWrkThrd == 1 && pThis->bInactivityGuard == 1);
- /* inactivity guard prevents shutdown of all workers while one should be running due to race
- * condition. It can lead to one more worker running than desired, but that is acceptable. After
- * all, that worker will shutdown itself due to inactivity timeout. If, however, none were running
- * when one was required, processing could come to a halt. -- rgerhards, 2008-01-21
- */
-
+ pthread_cleanup_push(wtpWrkrExecCancelCleanup, pWti);
+ wtiWorker(pWti);
pthread_cleanup_pop(0);
- pThis->iCurNumWrkThrd--;
- wtpSignalWrkrTermination(pThis);
-
- dbgprintf("%s: Worker thread %lx, terminated, num workers now %d\n",
- wtpGetDbgHdr(pThis), (unsigned long) pWti, pThis->iCurNumWrkThrd);
-
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
+ wtpWrkrExecCleanup(pWti);
ENDfunc
+ /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main
+ * thread after the broadcast, which could destroy the debug class, resulting in a potential
+ * segfault. So we need to do the broadcast as actually the last action in our processing
+ */
+ pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */
pthread_exit(0);
}
#pragma GCC diagnostic warning "-Wempty-body"
@@ -505,27 +401,20 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in
/* start a new worker */
static rsRetVal
-wtpStartWrkr(wtp_t *pThis, int bLockMutex)
+wtpStartWrkr(wtp_t *pThis)
{
- DEFiRet;
- DEFVARS_mutexProtection;
wti_t *pWti;
int i;
int iState;
+ DEFiRet;
ISOBJ_TYPE_assert(pThis, wtp);
- wtpProcessThrdChanges(pThis); // TODO: Performance: this causes a lot of FUTEX calls
+ d_pthread_mutex_lock(&pThis->mutWtp);
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
-
- pThis->iCurNumWrkThrd++;
-
- /* find free spot in thread table. If we find at least one worker that is in initialization,
- * we do NOT start a new one. Let's give the other one a chance, first.
- */
+ /* find free spot in thread table. */
for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) {
- if(wtiGetState(pThis->pWrkr[i], LOCK_MUTEX) == eWRKTHRD_STOPPED) {
+ if(wtiGetState(pThis->pWrkr[i]) == WRKTHRD_STOPPED) {
break;
}
}
@@ -533,17 +422,21 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex)
if(i == pThis->iNumWorkerThreads)
ABORT_FINALIZE(RS_RET_NO_MORE_THREADS);
+ if(i == 0 || pThis->toWrkShutdown == -1) {
+ wtiSetAlwaysRunning(pThis->pWrkr[i]);
+ }
+
pWti = pThis->pWrkr[i];
- wtiSetState(pWti, eWRKTHRD_RUN_CREATED, 0, LOCK_MUTEX);
- iState = pthread_create(&(pWti->thrdID), NULL, wtpWorker, (void*) pWti);
- dbgprintf("%s: started with state %d, num workers now %d\n",
- wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd);
+ wtiSetState(pWti, WRKTHRD_RUNNING);
+ iState = pthread_create(&(pWti->thrdID), &pThis->attrThrd, wtpWorker, (void*) pWti);
+ ATOMIC_INC(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd); /* we got one more! */
- /* indicate we just started a worker and would like to see it running */
- wtpSetInactivityGuard(pThis, 1, MUTEX_ALREADY_LOCKED);
+ DBGPRINTF("%s: started with state %d, num workers now %d\n",
+ wtpGetDbgHdr(pThis), iState,
+ ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd));
finalize_it:
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
+ d_pthread_mutex_unlock(&pThis->mutWtp);
RETiRet;
}
@@ -560,7 +453,6 @@ rsRetVal
wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr)
{
DEFiRet;
- DEFVARS_mutexProtection;
int nMissing; /* number workers missing to run */
int i;
@@ -569,29 +461,24 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr)
if(nMaxWrkr == 0)
FINALIZE;
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX);
-
if(nMaxWrkr > pThis->iNumWorkerThreads) /* limit to configured maximum */
nMaxWrkr = pThis->iNumWorkerThreads;
- nMissing = nMaxWrkr - pThis->iCurNumWrkThrd;
+ nMissing = nMaxWrkr - ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd);
if(nMissing > 0) {
- dbgprintf("%s: high activity - starting %d additional worker thread(s).\n", wtpGetDbgHdr(pThis), nMissing);
+ DBGPRINTF("%s: high activity - starting %d additional worker thread(s).\n",
+ wtpGetDbgHdr(pThis), nMissing);
/* start the rqtd nbr of workers */
for(i = 0 ; i < nMissing ; ++i) {
- CHKiRet(wtpStartWrkr(pThis, MUTEX_ALREADY_LOCKED));
- }
- } else {
- if(nMaxWrkr > 0) {
- dbgprintf("wtpAdviseMaxWorkers signals busy\n");
- wtpWakeupWrkr(pThis);
+ CHKiRet(wtpStartWrkr(pThis));
}
+ } else {
+ pthread_cond_signal(pThis->pcondBusy);
}
finalize_it:
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
RETiRet;
}
@@ -605,35 +492,9 @@ DEFpropSetMethPTR(wtp, pmutUsr, pthread_mutex_t)
DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t)
DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int))
DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*))
-DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int))
-DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int))
-DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int))
-DEFpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*, void*))
-DEFpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*))
-DEFpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*))
-
-
-/* return the current number of worker threads.
- * TODO: atomic operation would bring a nice performance
- * enhancemcent
- * rgerhards, 2008-01-27
- */
-int
-wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex)
-{
- DEFVARS_mutexProtection;
- int iNumWrkr;
-
- BEGINfunc
- ISOBJ_TYPE_assert(pThis, wtp);
-
- BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
- iNumWrkr = pThis->iCurNumWrkThrd;
- END_MTX_PROTECTED_OPERATIONS(&pThis->mut);
-
- ENDfunc
- return iNumWrkr;
-}
+DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*))
+DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*))
+DEFpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*))
/* set the debug header message
@@ -657,7 +518,7 @@ wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg)
pThis->pszDbgHdr = NULL;
}
- if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL)
+ if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL)
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */
@@ -687,6 +548,5 @@ BEGINObjClassInit(wtp, 1, OBJ_IS_CORE_MODULE)
CHKiRet(objUse(glbl, CORE_COMPONENT));
ENDObjClassInit(wtp)
-/*
- * vi:set ai:
+/* vi:set ai:
*/
diff --git a/runtime/wtp.h b/runtime/wtp.h
index 640c3320..7e6b4394 100644
--- a/runtime/wtp.h
+++ b/runtime/wtp.h
@@ -28,18 +28,9 @@
#include "obj.h"
#include "atomic.h"
-/* commands and states for worker threads. */
-typedef enum {
- eWRKTHRD_STOPPED = 0, /* worker thread is not running (either actually never ran or was shut down) */
- eWRKTHRD_TERMINATING = 1,/* worker thread has shut down, but some finalzing is still needed */
- /* ALL active states MUST be numerically higher than eWRKTHRD_TERMINATED and NONE must be lower! */
- eWRKTHRD_RUN_CREATED = 2,/* worker thread has been created, but not yet begun initialization (prob. not yet scheduled) */
- eWRKTHRD_RUN_INIT = 3, /* worker thread is initializing, but not yet fully running */
- eWRKTHRD_RUNNING = 4, /* worker thread is up and running and shall continue to do so */
- eWRKTHRD_SHUTDOWN = 5, /* worker thread is running but shall terminate when wtp is empty */
- eWRKTHRD_SHUTDOWN_IMMEDIATE = 6/* worker thread is running but shall terminate even if wtp is full */
- /* SHUTDOWN_IMMEDIATE MUST alsways be the numerically highest state! */
-} qWrkCmd_t;
+/* states for worker threads. */
+#define WRKTHRD_STOPPED FALSE
+#define WRKTHRD_RUNNING TRUE
/* possible states of a worker thread pool */
@@ -51,37 +42,33 @@ typedef enum {
/* the worker thread pool (wtp) object */
-typedef struct wtp_s {
+struct wtp_s {
BEGINobjInstance;
wtpState_t wtpState;
int iNumWorkerThreads;/* number of worker threads to use */
int iCurNumWrkThrd;/* current number of active worker threads */
struct wti_s **pWrkr;/* array with control structure for the worker thread(s) associated with this wtp */
int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */
- bool bInactivityGuard;/* prevents inactivity due to race condition */
rsRetVal (*pConsumer)(void *); /* user-supplied consumer function for dewtpd messages */
/* synchronization variables */
- pthread_mutex_t mutThrdShutdwn; /* mutex to guard thread shutdown processing */
- pthread_mutex_t mut; /* mutex for the wtp's thread management */
+ pthread_mutex_t mutWtp; /* mutex for the wtp's thread management */
pthread_cond_t condThrdTrm;/* signalled when threads terminate */
- int bThrdStateChanged; /* at least one thread state has changed if 1 */
/* end sync variables */
/* user objects */
- void *pUsr; /* pointer to user object */
+ void *pUsr; /* pointer to user object (in this case, the queue the wtp belongs to) */
+ pthread_attr_t attrThrd;/* attribute for new threads (created just once and cached here) */
pthread_mutex_t *pmutUsr;
pthread_cond_t *pcondBusy; /* condition the user will signal "busy again, keep runing" on (awakes worker) */
rsRetVal (*pfChkStopWrkr)(void *pUsr, int);
+ rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */
+ rsRetVal (*pfObjProcessed)(void *pUsr, wti_t *pWti); /* indicate user object is processed */
rsRetVal (*pfRateLimiter)(void *pUsr);
- rsRetVal (*pfIsIdle)(void *pUsr, int);
- rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int);
- rsRetVal (*pfOnIdle)(void *pUsr, int);
- rsRetVal (*pfOnWorkerCancel)(void *pUsr, void*pWti);
- rsRetVal (*pfOnWorkerStartup)(void *pUsr);
- rsRetVal (*pfOnWorkerShutdown)(void *pUsr);
+ rsRetVal (*pfDoWork)(void *pUsr, void *pWti);
/* end user objects */
uchar *pszDbgHdr; /* header string for debug messages */
- DEF_ATOMIC_HELPER_MUT(mutThrdStateChanged);
-} wtp_t;
+ DEF_ATOMIC_HELPER_MUT(mutCurNumWrkThrd);
+ DEF_ATOMIC_HELPER_MUT(mutWtpState);
+};
/* some symbolic constants for easier reference */
@@ -92,26 +79,18 @@ rsRetVal wtpConstructFinalize(wtp_t *pThis);
rsRetVal wtpDestruct(wtp_t **ppThis);
rsRetVal wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr);
rsRetVal wtpProcessThrdChanges(wtp_t *pThis);
-rsRetVal wtpSetInactivityGuard(wtp_t *pThis, int bNewState, int bLockMutex);
-rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex);
+rsRetVal wtpChkStopWrkr(wtp_t *pThis, int bLockUsrMutex);
rsRetVal wtpSetState(wtp_t *pThis, wtpState_t iNewState);
-rsRetVal wtpWakeupWrkr(wtp_t *pThis);
rsRetVal wtpWakeupAllWrkr(wtp_t *pThis);
rsRetVal wtpCancelAll(wtp_t *pThis);
rsRetVal wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg);
-rsRetVal wtpSignalWrkrTermination(wtp_t *pWtp);
rsRetVal wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout);
-int wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex);
-void wtpSetThrdStateChanged(wtp_t *pThis, int val);
PROTOTYPEObjClassInit(wtp);
PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int));
PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*));
-PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int));
-PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int));
-PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int));
-PROTOTYPEpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*,void*));
-PROTOTYPEpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*));
-PROTOTYPEpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*));
+PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*));
+PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*));
+PROTOTYPEpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*));
PROTOTYPEpropSetMeth(wtp, toWrkShutdown, long);
PROTOTYPEpropSetMeth(wtp, wtpState, wtpState_t);
PROTOTYPEpropSetMeth(wtp, iMaxWorkerThreads, int);
diff --git a/runtime/zlibw.c b/runtime/zlibw.c
index 2b386213..455c20d9 100644
--- a/runtime/zlibw.c
+++ b/runtime/zlibw.c
@@ -34,6 +34,7 @@
#include "zlibw.h"
MODULE_TYPE_LIB
+MODULE_TYPE_NOKEEP
/* static data */
DEFobjStaticHelpers