From 41327992680cfb8dd602ea92d6902448dd66413d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 7 Sep 2010 13:06:04 +0200 Subject: acquire /dev/log socket optionally from systemd --- runtime/Makefile.am | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'runtime') diff --git a/runtime/Makefile.am b/runtime/Makefile.am index f7db3e35..ef119492 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -81,6 +81,8 @@ librsyslog_la_SOURCES = \ prop.h \ cfsysline.c \ cfsysline.h \ + sd-daemon.c \ + sd-daemon.h \ \ \ ../action.h \ @@ -177,3 +179,6 @@ lmnsd_gtls_la_LDFLAGS = -module -avoid-version lmnsd_gtls_la_LIBADD = $(GNUTLS_LIBS) endif +update-systemd: + curl http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c > sd-daemon.c + curl http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h > sd-daemon.h -- cgit From 3e2d011e8c1c42801194d387b75ca4f4a175ae5d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Sep 2010 14:19:05 +0200 Subject: added forgotten files --- runtime/sd-daemon.c | 448 ++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/sd-daemon.h | 257 ++++++++++++++++++++++++++++++ 2 files changed, 705 insertions(+) create mode 100644 runtime/sd-daemon.c create mode 100644 runtime/sd-daemon.h (limited to 'runtime') diff --git a/runtime/sd-daemon.c b/runtime/sd-daemon.c new file mode 100644 index 00000000..5df70e38 --- /dev/null +++ b/runtime/sd-daemon.c @@ -0,0 +1,448 @@ +/*-*- Mode: C; c-basic-offset: 8 -*-*/ + +/*** + Copyright 2010 Lennart Poettering + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +***/ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sd-daemon.h" + +int sd_listen_fds(int unset_environment) { + +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + int r, fd; + const char *e; + char *p = NULL; + unsigned long l; + + if (!(e = getenv("LISTEN_PID"))) { + r = 0; + goto finish; + } + + errno = 0; + l = strtoul(e, &p, 10); + + if (errno != 0) { + r = -errno; + goto finish; + } + + if (!p || *p || l <= 0) { + r = -EINVAL; + goto finish; + } + + /* Is this for us? */ + if (getpid() != (pid_t) l) { + r = 0; + goto finish; + } + + if (!(e = getenv("LISTEN_FDS"))) { + r = 0; + goto finish; + } + + errno = 0; + l = strtoul(e, &p, 10); + + if (errno != 0) { + r = -errno; + goto finish; + } + + if (!p || *p) { + r = -EINVAL; + goto finish; + } + + for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + (int) l; fd ++) { + int flags; + + if ((flags = fcntl(fd, F_GETFD)) < 0) { + r = -errno; + goto finish; + } + + if (flags & FD_CLOEXEC) + continue; + + if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) { + r = -errno; + goto finish; + } + } + + r = (int) l; + +finish: + if (unset_environment) { + unsetenv("LISTEN_PID"); + unsetenv("LISTEN_FDS"); + } + + return r; +#endif +} + +int sd_is_fifo(int fd, const char *path) { + struct stat st_fd; + + if (fd < 0) + return -EINVAL; + + memset(&st_fd, 0, sizeof(st_fd)); + if (fstat(fd, &st_fd) < 0) + return -errno; + + if (!S_ISFIFO(st_fd.st_mode)) + return 0; + + if (path) { + struct stat st_path; + + memset(&st_path, 0, sizeof(st_path)); + if (stat(path, &st_path) < 0) { + + if (errno == ENOENT || errno == ENOTDIR) + return 0; + + return -errno; + } + + return + st_path.st_dev == st_fd.st_dev && + st_path.st_ino == st_fd.st_ino; + } + + return 1; +} + +static int sd_is_socket_internal(int fd, int type, int listening) { + struct stat st_fd; + + if (fd < 0 || type < 0) + return -EINVAL; + + if (fstat(fd, &st_fd) < 0) + return -errno; + + if (!S_ISSOCK(st_fd.st_mode)) + return 0; + + if (type != 0) { + int other_type = 0; + socklen_t l = sizeof(other_type); + + if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &other_type, &l) < 0) + return -errno; + + if (l != sizeof(other_type)) + return -EINVAL; + + if (other_type != type) + return 0; + } + + if (listening >= 0) { + int accepting = 0; + socklen_t l = sizeof(accepting); + + if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &accepting, &l) < 0) + return -errno; + + if (l != sizeof(accepting)) + return -EINVAL; + + if (!accepting != !listening) + return 0; + } + + return 1; +} + +union sockaddr_union { + struct sockaddr sa; + struct sockaddr_in in4; + struct sockaddr_in6 in6; + struct sockaddr_un un; + struct sockaddr_storage storage; +}; + +int sd_is_socket(int fd, int family, int type, int listening) { + int r; + + if (family < 0) + return -EINVAL; + + if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) + return r; + + if (family > 0) { + union sockaddr_union sockaddr; + socklen_t l; + + memset(&sockaddr, 0, sizeof(sockaddr)); + l = sizeof(sockaddr); + + if (getsockname(fd, &sockaddr.sa, &l) < 0) + return -errno; + + if (l < sizeof(sa_family_t)) + return -EINVAL; + + return sockaddr.sa.sa_family == family; + } + + return 1; +} + +int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) { + union sockaddr_union sockaddr; + socklen_t l; + int r; + + if (family != 0 && family != AF_INET && family != AF_INET6) + return -EINVAL; + + if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) + return r; + + memset(&sockaddr, 0, sizeof(sockaddr)); + l = sizeof(sockaddr); + + if (getsockname(fd, &sockaddr.sa, &l) < 0) + return -errno; + + if (l < sizeof(sa_family_t)) + return -EINVAL; + + if (sockaddr.sa.sa_family != AF_INET && + sockaddr.sa.sa_family != AF_INET6) + return 0; + + if (family > 0) + if (sockaddr.sa.sa_family != family) + return 0; + + if (port > 0) { + if (sockaddr.sa.sa_family == AF_INET) { + if (l < sizeof(struct sockaddr_in)) + return -EINVAL; + + return htons(port) == sockaddr.in4.sin_port; + } else { + if (l < sizeof(struct sockaddr_in6)) + return -EINVAL; + + return htons(port) == sockaddr.in6.sin6_port; + } + } + + return 1; +} + +int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) { + union sockaddr_union sockaddr; + socklen_t l; + int r; + + if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) + return r; + + memset(&sockaddr, 0, sizeof(sockaddr)); + l = sizeof(sockaddr); + + if (getsockname(fd, &sockaddr.sa, &l) < 0) + return -errno; + + if (l < sizeof(sa_family_t)) + return -EINVAL; + + if (sockaddr.sa.sa_family != AF_UNIX) + return 0; + + if (path) { + if (length <= 0) + length = strlen(path); + + if (length <= 0) + /* Unnamed socket */ + return l == sizeof(sa_family_t); + + if (path[0]) + /* Normal path socket */ + return + (l >= sizeof(sa_family_t) + length + 1) && + memcmp(path, sockaddr.un.sun_path, length+1) == 0; + else + /* Abstract namespace socket */ + return + (l == sizeof(sa_family_t) + length) && + memcmp(path, sockaddr.un.sun_path, length) == 0; + } + + return 1; +} + +int sd_notify(int unset_environment, const char *state) { +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + int fd = -1, r; + struct msghdr msghdr; + struct iovec iovec; + union sockaddr_union sockaddr; + struct ucred *ucred; + union { + struct cmsghdr cmsghdr; + uint8_t buf[CMSG_SPACE(sizeof(struct ucred))]; + } control; + const char *e; + + if (!state) { + r = -EINVAL; + goto finish; + } + + if (!(e = getenv("NOTIFY_SOCKET"))) + return 0; + + /* Must be an abstract socket, or an absolute path */ + if ((e[0] != '@' && e[0] != '/') || e[1] == 0) { + r = -EINVAL; + goto finish; + } + + if ((fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) { + r = -errno; + goto finish; + } + + memset(&sockaddr, 0, sizeof(sockaddr)); + sockaddr.sa.sa_family = AF_UNIX; + strncpy(sockaddr.un.sun_path, e, sizeof(sockaddr.un.sun_path)); + + if (sockaddr.un.sun_path[0] == '@') + sockaddr.un.sun_path[0] = 0; + + memset(&iovec, 0, sizeof(iovec)); + iovec.iov_base = (char*) state; + iovec.iov_len = strlen(state); + + memset(&control, 0, sizeof(control)); + control.cmsghdr.cmsg_level = SOL_SOCKET; + control.cmsghdr.cmsg_type = SCM_CREDENTIALS; + control.cmsghdr.cmsg_len = CMSG_LEN(sizeof(struct ucred)); + + ucred = (struct ucred*) CMSG_DATA(&control.cmsghdr); + ucred->pid = getpid(); + ucred->uid = getuid(); + ucred->gid = getgid(); + + memset(&msghdr, 0, sizeof(msghdr)); + msghdr.msg_name = &sockaddr; + msghdr.msg_namelen = sizeof(struct sockaddr_un); + msghdr.msg_iov = &iovec; + msghdr.msg_iovlen = 1; + msghdr.msg_control = &control; + msghdr.msg_controllen = control.cmsghdr.cmsg_len; + + if (sendmsg(fd, &msghdr, MSG_NOSIGNAL) < 0) { + r = -errno; + goto finish; + } + + r = 1; + +finish: + if (unset_environment) + unsetenv("NOTIFY_SOCKET"); + + if (fd >= 0) + close(fd); + + return r; +#endif +} + +int sd_notifyf(int unset_environment, const char *format, ...) { +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + va_list ap; + char *p = NULL; + int r; + + va_start(ap, format); + r = vasprintf(&p, format, ap); + va_end(ap); + + if (r < 0 || !p) + return -ENOMEM; + + r = sd_notify(unset_environment, p); + free(p); + + return r; +#endif +} + +int sd_booted(void) { +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) + return 0; +#else + + struct stat a, b; + + /* We simply test whether the systemd cgroup hierarchy is + * mounted */ + + if (lstat("/cgroup", &a) < 0) + return 0; + + if (lstat("/cgroup/systemd", &b) < 0) + return 0; + + return a.st_dev != b.st_dev; +#endif +} diff --git a/runtime/sd-daemon.h b/runtime/sd-daemon.h new file mode 100644 index 00000000..fd6221f2 --- /dev/null +++ b/runtime/sd-daemon.h @@ -0,0 +1,257 @@ +/*-*- Mode: C; c-basic-offset: 8 -*-*/ + +#ifndef foosddaemonhfoo +#define foosddaemonhfoo + +/*** + Copyright 2010 Lennart Poettering + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +***/ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + Reference implementation of a few systemd related interfaces for + writing daemons. These interfaces are trivial to implement. To + simplify porting we provide this reference implementation. + Applications are welcome to reimplement the algorithms described + here if they do not want to include these two source files. + + The following functionality is provided: + + - Support for logging with log levels on stderr + - File descriptor passing for socket-based activation + - Daemon startup and status notification + - Detection of systemd boots + + You may compile this with -DDISABLE_SYSTEMD to disable systemd + support. This makes all those calls NOPs that are directly related to + systemd (i.e. only sd_is_xxx() will stay useful). + + Since this is drop-in code we don't want any of our symbols to be + exported in any case. Hence we declare hidden visibility for all of + them. + + You may find an up-to-date version of these source files online: + + http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h + http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c + + This should compile on non-Linux systems, too, but with the + exception of the sd_is_xxx() calls all functions will become NOPs. + + See sd-daemon(7) for more information. +*/ + +#if __GNUC__ >= 4 +#define _sd_printf_attr_(a,b) __attribute__ ((format (printf, a, b))) +#define _sd_hidden_ __attribute__ ((visibility("hidden"))) +#else +#define _sd_printf_attr_(a,b) +#define _sd_hidden_ +#endif + +/* + Log levels for usage on stderr: + + fprintf(stderr, SD_NOTICE "Hello World!\n"); + + This is similar to printk() usage in the kernel. +*/ +#define SD_EMERG "<0>" /* system is unusable */ +#define SD_ALERT "<1>" /* action must be taken immediately */ +#define SD_CRIT "<2>" /* critical conditions */ +#define SD_ERR "<3>" /* error conditions */ +#define SD_WARNING "<4>" /* warning conditions */ +#define SD_NOTICE "<5>" /* normal but significant condition */ +#define SD_INFO "<6>" /* informational */ +#define SD_DEBUG "<7>" /* debug-level messages */ + +/* The first passed file descriptor is fd 3 */ +#define SD_LISTEN_FDS_START 3 + +/* + Returns how many file descriptors have been passed, or a negative + errno code on failure. Optionally, removes the $LISTEN_FDS and + $LISTEN_PID file descriptors from the environment (recommended, but + problematic in threaded environments). If r is the return value of + this function you'll find the file descriptors passed as fds + SD_LISTEN_FDS_START to SD_LISTEN_FDS_START+r-1. Returns a negative + errno style error code on failure. This function call ensures that + the FD_CLOEXEC flag is set for the passed file descriptors, to make + sure they are not passed on to child processes. If FD_CLOEXEC shall + not be set, the caller needs to unset it after this call for all file + descriptors that are used. + + See sd_listen_fds(3) for more information. +*/ +int sd_listen_fds(int unset_environment) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is a FIFO in the file system stored under the + specified path, 0 otherwise. If path is NULL a path name check will + not be done and the call only verifies if the file descriptor + refers to a FIFO. Returns a negative errno style error code on + failure. + + See sd_is_fifo(3) for more information. +*/ +int sd_is_fifo(int fd, const char *path) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is a socket of the specified family (AF_INET, + ...) and type (SOCK_DGRAM, SOCK_STREAM, ...), 0 otherwise. If + family is 0 a socket family check will not be done. If type is 0 a + socket type check will not be done and the call only verifies if + the file descriptor refers to a socket. If listening is > 0 it is + verified that the socket is in listening mode. (i.e. listen() has + been called) If listening is == 0 it is verified that the socket is + not in listening mode. If listening is < 0 no listening mode check + is done. Returns a negative errno style error code on failure. + + See sd_is_socket(3) for more information. +*/ +int sd_is_socket(int fd, int family, int type, int listening) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is an Internet socket, of the specified family + (either AF_INET or AF_INET6) and the specified type (SOCK_DGRAM, + SOCK_STREAM, ...), 0 otherwise. If version is 0 a protocol version + check is not done. If type is 0 a socket type check will not be + done. If port is 0 a socket port check will not be done. The + listening flag is used the same way as in sd_is_socket(). Returns a + negative errno style error code on failure. + + See sd_is_socket_inet(3) for more information. +*/ +int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) _sd_hidden_; + +/* + Helper call for identifying a passed file descriptor. Returns 1 if + the file descriptor is an AF_UNIX socket of the specified type + (SOCK_DGRAM, SOCK_STREAM, ...) and path, 0 otherwise. If type is 0 + a socket type check will not be done. If path is NULL a socket path + check will not be done. For normal AF_UNIX sockets set length to + 0. For abstract namespace sockets set length to the length of the + socket name (including the initial 0 byte), and pass the full + socket path in path (including the initial 0 byte). The listening + flag is used the same way as in sd_is_socket(). Returns a negative + errno style error code on failure. + + See sd_is_socket_unix(3) for more information. +*/ +int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) _sd_hidden_; + +/* + Informs systemd about changed daemon state. This takes a number of + newline seperated environment-style variable assignments in a + string. The following variables are known: + + READY=1 Tells systemd that daemon startup is finished (only + relevant for services of Type=notify). The passed + argument is a boolean "1" or "0". Since there is + little value in signalling non-readiness the only + value daemons should send is "READY=1". + + STATUS=... Passes a single-line status string back to systemd + that describes the daemon state. This is free-from + and can be used for various purposes: general state + feedback, fsck-like programs could pass completion + percentages and failing programs could pass a human + readable error message. Example: "STATUS=Completed + 66% of file system check..." + + ERRNO=... If a daemon fails, the errno-style error code, + formatted as string. Example: "ERRNO=2" for ENOENT. + + BUSERROR=... If a daemon fails, the D-Bus error-style error + code. Example: "BUSERROR=org.freedesktop.DBus.Error.TimedOut" + + MAINPID=... The main pid of a daemon, in case systemd did not + fork off the process itself. Example: "MAINPID=4711" + + Daemons can choose to send additional variables. However, it is + recommened to prefix variable names not listed above with X_. + + Returns a negative errno-style error code on failure. Returns > 0 + if systemd could be notified, 0 if it couldn't possibly because + systemd is not running. + + Example: When a daemon finished starting up, it could issue this + call to notify systemd about it: + + sd_notify(0, "READY=1"); + + See sd_notifyf() for more complete examples. + + See sd_notify(3) for more information. +*/ +int sd_notify(int unset_environment, const char *state) _sd_hidden_; + +/* + Similar to sd_notify() but takes a format string. + + Example 1: A daemon could send the following after initialization: + + sd_notifyf(0, "READY=1\n" + "STATUS=Processing requests...\n" + "MAINPID=%lu", + (unsigned long) getpid()); + + Example 2: A daemon could send the following shortly before + exiting, on failure: + + sd_notifyf(0, "STATUS=Failed to start up: %s\n" + "ERRNO=%i", + strerror(errno), + errno); + + See sd_notifyf(3) for more information. +*/ +int sd_notifyf(int unset_environment, const char *format, ...) _sd_printf_attr_(2,3) _sd_hidden_; + +/* + Returns > 0 if the system was booted with systemd. Returns < 0 on + error. Returns 0 if the system was not booted with systemd. Note + that all of the functions above handle non-systemd boots just + fine. You should NOT protect them with a call to this function. Also + note that this function checks whether the system, not the user + session is controlled by systemd. However the functions above work + for both session and system services. + + See sd_booted(3) for more information. +*/ +int sd_booted(void) _sd_hidden_; + +#ifdef __cplusplus +} +#endif + +#endif -- cgit From 21a2672222a1b403ff0811154dec156676bed732 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Sep 2010 12:46:03 +0200 Subject: moved systemd interface to rsyslog convenience lib Mostly a refresh of sd-daemon.[ch] from its source plus some make file changes. We now have systemd interfaces inside rsyslog, so that all plugins interested can call the interfaces. Seems not to be totally necessary right now, but will help in the long term. --- runtime/Makefile.am | 4 ++-- runtime/sd-daemon.c | 31 +++++++++---------------------- runtime/sd-daemon.h | 12 ++++++++---- 3 files changed, 19 insertions(+), 28 deletions(-) (limited to 'runtime') diff --git a/runtime/Makefile.am b/runtime/Makefile.am index ef119492..5b2598b2 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -101,9 +101,9 @@ librsyslog_la_SOURCES = \ # runtime or will no longer be needed. -- rgerhards, 2008-06-13 if WITH_MODDIRS -librsyslog_la_CPPFLAGS = -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS) +librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS) else -librsyslog_la_CPPFLAGS = -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS) +librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS) endif #librsyslog_la_LDFLAGS = -module -avoid-version librsyslog_la_LIBADD = $(DL_LIBS) $(RT_LIBS) diff --git a/runtime/sd-daemon.c b/runtime/sd-daemon.c index 5df70e38..9c23b917 100644 --- a/runtime/sd-daemon.c +++ b/runtime/sd-daemon.c @@ -1,4 +1,4 @@ -/*-*- Mode: C; c-basic-offset: 8 -*-*/ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ /*** Copyright 2010 Lennart Poettering @@ -325,18 +325,13 @@ int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t } int sd_notify(int unset_environment, const char *state) { -#if defined(DISABLE_SYSTEMD) || !defined(__linux__) +#if defined(DISABLE_SYSTEMD) || !defined(__linux__) || !defined(SOCK_CLOEXEC) return 0; #else int fd = -1, r; struct msghdr msghdr; struct iovec iovec; union sockaddr_union sockaddr; - struct ucred *ucred; - union { - struct cmsghdr cmsghdr; - uint8_t buf[CMSG_SPACE(sizeof(struct ucred))]; - } control; const char *e; if (!state) { @@ -369,23 +364,15 @@ int sd_notify(int unset_environment, const char *state) { iovec.iov_base = (char*) state; iovec.iov_len = strlen(state); - memset(&control, 0, sizeof(control)); - control.cmsghdr.cmsg_level = SOL_SOCKET; - control.cmsghdr.cmsg_type = SCM_CREDENTIALS; - control.cmsghdr.cmsg_len = CMSG_LEN(sizeof(struct ucred)); - - ucred = (struct ucred*) CMSG_DATA(&control.cmsghdr); - ucred->pid = getpid(); - ucred->uid = getuid(); - ucred->gid = getgid(); - memset(&msghdr, 0, sizeof(msghdr)); msghdr.msg_name = &sockaddr; - msghdr.msg_namelen = sizeof(struct sockaddr_un); + msghdr.msg_namelen = sizeof(sa_family_t) + strlen(e); + + if (msghdr.msg_namelen > sizeof(struct sockaddr_un)) + msghdr.msg_namelen = sizeof(struct sockaddr_un); + msghdr.msg_iov = &iovec; msghdr.msg_iovlen = 1; - msghdr.msg_control = &control; - msghdr.msg_controllen = control.cmsghdr.cmsg_len; if (sendmsg(fd, &msghdr, MSG_NOSIGNAL) < 0) { r = -errno; @@ -437,10 +424,10 @@ int sd_booted(void) { /* We simply test whether the systemd cgroup hierarchy is * mounted */ - if (lstat("/cgroup", &a) < 0) + if (lstat("/sys/fs/cgroup", &a) < 0) return 0; - if (lstat("/cgroup/systemd", &b) < 0) + if (lstat("/sys/fs/cgroup/systemd", &b) < 0) return 0; return a.st_dev != b.st_dev; diff --git a/runtime/sd-daemon.h b/runtime/sd-daemon.h index fd6221f2..45aac8bd 100644 --- a/runtime/sd-daemon.h +++ b/runtime/sd-daemon.h @@ -1,4 +1,4 @@ -/*-*- Mode: C; c-basic-offset: 8 -*-*/ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ #ifndef foosddaemonhfoo #define foosddaemonhfoo @@ -67,9 +67,13 @@ extern "C" { See sd-daemon(7) for more information. */ -#if __GNUC__ >= 4 +#if (__GNUC__ >= 4) #define _sd_printf_attr_(a,b) __attribute__ ((format (printf, a, b))) -#define _sd_hidden_ __attribute__ ((visibility("hidden"))) +# if defined(SD_EXPORT_SYMBOLS) +# define _sd_hidden_ +# else +# define _sd_hidden_ __attribute__ ((visibility("hidden"))) +# endif #else #define _sd_printf_attr_(a,b) #define _sd_hidden_ @@ -171,7 +175,7 @@ int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t /* Informs systemd about changed daemon state. This takes a number of - newline seperated environment-style variable assignments in a + newline separated environment-style variable assignments in a string. The following variables are known: READY=1 Tells systemd that daemon startup is finished (only -- cgit From e86cb62f1299ef18732f7b3b87d45a840ee38f1e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 13 Sep 2010 15:43:56 +0200 Subject: improved statistics-gathering subsystem ... well, actually this is a first real implementation of this subsystem. I have added a counter registry, a way to access the countres (as readable string) and a way to define and maintem them. Also, module impstats has been updated to utilize the new system. Finally, I added some counters. I hope that this sets the baseline for useful future enhancements. --- runtime/Makefile.am | 6 +- runtime/queue.c | 32 ++++++ runtime/queue.h | 6 + runtime/rsyslog.c | 3 + runtime/rsyslog.h | 2 + runtime/statsobj.c | 315 ++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/statsobj.h | 124 +++++++++++++++++++++ 7 files changed, 486 insertions(+), 2 deletions(-) create mode 100644 runtime/statsobj.c create mode 100644 runtime/statsobj.h (limited to 'runtime') diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 5b2598b2..01b224e7 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -45,6 +45,8 @@ librsyslog_la_SOURCES = \ modules.h \ apc.c \ apc.h \ + statsobj.c \ + statsobj.h \ sync.c \ sync.h \ expr.c \ @@ -81,8 +83,8 @@ librsyslog_la_SOURCES = \ prop.h \ cfsysline.c \ cfsysline.h \ - sd-daemon.c \ - sd-daemon.h \ + sd-daemon.c \ + sd-daemon.h \ \ \ ../action.h \ diff --git a/runtime/queue.c b/runtime/queue.c index 60d17086..387c15c2 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -58,6 +58,7 @@ #include "errmsg.h" #include "datetime.h" #include "unicode-helper.h" +#include "statsobj.h" #include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS @@ -70,6 +71,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(strm) DEFobjCurrIf(errmsg) DEFobjCurrIf(datetime) +DEFobjCurrIf(statsobj) /* forward-definitions */ static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); @@ -1817,6 +1819,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; uchar pszBuf[64]; + uchar *qName; size_t lenBuf; ASSERT(pThis != NULL); @@ -1886,6 +1889,27 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ qqueueAdviseMaxWorkers(pThis); pThis->bQueueStarted = 1; + /* support statistics gathering */ + qName = obj.GetName((obj_t*)pThis); + CHKiRet(statsobj.Construct(&pThis->statsobj)); + CHKiRet(statsobj.SetName(pThis->statsobj, qName)); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("size"), + ctrType_Int, &pThis->iQueueSize)); + + STATSCOUNTER_INIT(pThis->ctrEnqueued, pThis->mutCtrEnqueued); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("enqueued"), + ctrType_IntCtr, &pThis->ctrEnqueued)); + + STATSCOUNTER_INIT(pThis->ctrFull, pThis->mutCtrFull); + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("full"), + ctrType_IntCtr, &pThis->ctrFull)); + + pThis->ctrMaxqsize = 0; + CHKiRet(statsobj.AddCounter(pThis->statsobj, UCHAR_CONSTANT("maxqsize"), + ctrType_Int, &pThis->ctrMaxqsize)); + + CHKiRet(statsobj.ConstructFinalize(pThis->statsobj)); + finalize_it: RETiRet; } @@ -2119,6 +2143,10 @@ CODESTARTobjDestruct(qqueue) free(pThis->pszFilePrefix); free(pThis->pszSpoolDir); + + /* some queues do not provide stats and thus have no statsobj! */ + if(pThis->statsobj != NULL) + statsobj.Destruct(&pThis->statsobj); ENDobjDestruct(qqueue) @@ -2178,6 +2206,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) DEFiRet; struct timespec t; + STATSCOUNTER_INC(pThis->ctrEnqueued, pThis->mutCtrEnqueued); /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); @@ -2225,6 +2254,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); timeoutComp(&t, pThis->toEnq); + STATSCOUNTER_INC(pThis->ctrFull, pThis->mutCtrFull); // TODO : handle enqOnly => discard! if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); @@ -2235,6 +2265,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); + STATSCOUNTER_SETMAX_NOMUT(pThis->ctrMaxqsize, pThis->iQueueSize); finalize_it: RETiRet; @@ -2414,6 +2445,7 @@ BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/queue.h b/runtime/queue.h index 1c758134..38e248cd 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -29,6 +29,7 @@ #include "wtp.h" #include "batch.h" #include "stream.h" +#include "statsobj.h" /* support for the toDelete list */ typedef struct toDeleteLst_s toDeleteLst_t; @@ -165,6 +166,11 @@ struct queue_s { } tVars; DEF_ATOMIC_HELPER_MUT(mutQueueSize); DEF_ATOMIC_HELPER_MUT(mutLogDeq); + /* for statistics subsystem */ + statsobj_t *statsobj; + STATSCOUNTER_DEF(ctrEnqueued, mutCtrEnqueued); + STATSCOUNTER_DEF(ctrFull, mutCtrFull); + int ctrMaxqsize; }; diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index a9794840..8baa2b59 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -82,6 +82,7 @@ #include "ruleset.h" #include "parser.h" #include "strgen.h" +#include "statsobj.h" #include "atomic.h" /* forward definitions */ @@ -150,6 +151,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) * class immediately after it is initialized. And, of course, we load those classes * first that we use ourselfs... -- rgerhards, 2008-03-07 */ + if(ppErrObj != NULL) *ppErrObj = "statsobj"; + CHKiRet(statsobjClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "prop"; CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 34eaedca..7b2655b0 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -150,6 +150,8 @@ typedef struct parser_s parser_t; typedef struct parserList_s parserList_t; typedef struct strgen_s strgen_t; typedef struct strgenList_s strgenList_t; +typedef struct statsobj_s statsobj_t; +typedef struct statsctr_s statsctr_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ diff --git a/runtime/statsobj.c b/runtime/statsobj.c new file mode 100644 index 00000000..e1a89cf4 --- /dev/null +++ b/runtime/statsobj.c @@ -0,0 +1,315 @@ +/* The statsobj object. + * + * This object provides a statistics-gathering facility inside rsyslog. This + * functionality will be pragmatically implemented and extended. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include + +#include "rsyslog.h" +#include "unicode-helper.h" +#include "obj.h" +#include "statsobj.h" +#include "sysvar.h" +#include "srUtils.h" +#include "stringbuf.h" + + +/* externally-visiable data (see statsobj.h for explanation) */ +int GatherStats = 0; + +/* static data */ +DEFobjStaticHelpers + +/* doubly linked list of stats objects. Object is automatically linked to it + * upon construction. Enqueue always happens at the front (simplifies logic). + */ +static statsobj_t *objRoot = NULL; +static statsobj_t *objLast = NULL; + +static pthread_mutex_t mutStats; + +/* ------------------------------ statsobj linked list maintenance ------------------------------ */ + +static inline void +addToObjList(statsobj_t *pThis) +{ + pthread_mutex_lock(&mutStats); + pThis->prev = objLast; + if(objLast != NULL) + objLast->next = pThis; + objLast = pThis; + if(objRoot == NULL) + objRoot = pThis; + pthread_mutex_unlock(&mutStats); +} + + +static inline void +removeFromObjList(statsobj_t *pThis) +{ + pthread_mutex_lock(&mutStats); + if(pThis->prev != NULL) + pThis->prev->next = pThis->next; + if(pThis->next != NULL) + pThis->next->prev = pThis->prev; + if(objLast == pThis) + objLast = pThis->prev; + if(objRoot == pThis) + objRoot = pThis->next; + pthread_mutex_unlock(&mutStats); +} + + +static inline void +addCtrToList(statsobj_t *pThis, ctr_t *pCtr) +{ + pthread_mutex_lock(&pThis->mutCtr); + pCtr->prev = pThis->ctrLast; + if(pThis->ctrLast != NULL) + pThis->ctrLast->next = pCtr; + pThis->ctrLast = pCtr; + if(pThis->ctrRoot == NULL) + pThis->ctrRoot = pCtr; + pthread_mutex_unlock(&pThis->mutCtr); +} + +/* ------------------------------ methods ------------------------------ */ + + +/* Standard-Constructor + */ +BEGINobjConstruct(statsobj) /* be sure to specify the object type also in END macro! */ + pthread_mutex_init(&pThis->mutCtr, NULL); + pThis->ctrLast = NULL; + pThis->ctrRoot = NULL; +ENDobjConstruct(statsobj) + + +/* ConstructionFinalizer + */ +static rsRetVal +statsobjConstructFinalize(statsobj_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, statsobj); + addToObjList(pThis); + RETiRet; +} + + +/* set name. Note that we make our own copy of the memory, caller is + * responsible to free up name it passes in (if required). + */ +static rsRetVal +setName(statsobj_t *pThis, uchar *name) +{ + DEFiRet; + CHKmalloc(pThis->name = ustrdup(name)); +finalize_it: + RETiRet; +} + + +/* add a counter to an object + * ctrName is duplicated, caller must free it if requried + */ +static rsRetVal +addCounter(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr) +{ + ctr_t *ctr; + DEFiRet; + + CHKmalloc(ctr = malloc(sizeof(ctr_t))); + ctr->next = NULL; + ctr->prev = NULL; + CHKmalloc(ctr->name = ustrdup(ctrName)); + ctr->ctrType = ctrType; + switch(ctrType) { + case ctrType_IntCtr: + ctr->val.pIntCtr = (intctr_t*) pCtr; + break; + case ctrType_Int: + ctr->val.pInt = (int*) pCtr; + break; + } + addCtrToList(pThis, ctr); + +finalize_it: + RETiRet; +} + + +/* get all the object's countes together with object name as one line. + */ +static rsRetVal +getStatsLine(statsobj_t *pThis, cstr_t **ppcstr) +{ + cstr_t *pcstr; + ctr_t *pCtr; + DEFiRet; + + CHKiRet(cstrConstruct(&pcstr)); + rsCStrAppendStr(pcstr, pThis->name); + rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT(": "), 2); + + /* now add all counters to this line */ + pthread_mutex_lock(&pThis->mutCtr); + for(pCtr = pThis->ctrRoot ; pCtr != NULL ; pCtr = pCtr->next) { + rsCStrAppendStr(pcstr, pCtr->name); + cstrAppendChar(pcstr, '='); + switch(pCtr->ctrType) { + case ctrType_IntCtr: + rsCStrAppendInt(pcstr, *(pCtr->val.pIntCtr)); // TODO: OK????? + break; + case ctrType_Int: + rsCStrAppendInt(pcstr, *(pCtr->val.pInt)); + break; + } + cstrAppendChar(pcstr, ' '); + } + pthread_mutex_unlock(&pThis->mutCtr); + + CHKiRet(cstrFinalize(pcstr)); + *ppcstr = pcstr; + +finalize_it: + RETiRet; +} + + +/* this function can be used to obtain all stats lines. In this case, + * a callback must be provided. This module than iterates over all objects and + * submits each stats line to the callback. The callback has two parameters: + * the first one is a caller-provided void*, the second one the cstr_t with the + * line. If the callback reports an error, processing is stopped. + */ +static rsRetVal +getAllStatsLines(rsRetVal(*cb)(void*, cstr_t*), void *usrptr) +{ + statsobj_t *o; + cstr_t *cstr; + DEFiRet; + + for(o = objRoot ; o != NULL ; o = o->next) { + CHKiRet(getStatsLine(o, &cstr)); + CHKiRet(cb(usrptr, cstr)); + rsCStrDestruct(&cstr); + } + +finalize_it: + RETiRet; +} + + +/* Enable statistics gathering. currently there is no function to disable it + * again, as this is right now not needed. + */ +static rsRetVal +enableStats() +{ + GatherStats = 1; + return RS_RET_OK; +} + + +/* destructor for the statsobj object */ +BEGINobjDestruct(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ + ctr_t *ctr, *ctrToDel; +CODESTARTobjDestruct(statsobj) + removeFromObjList(pThis); + + /* destruct counters */ + ctr = pThis->ctrRoot; + while(ctr != NULL) { + ctrToDel = ctr; + ctr = ctr->next; + free(ctrToDel->name); + free(ctrToDel); + } + + pthread_mutex_destroy(&pThis->mutCtr); + free(pThis->name); +ENDobjDestruct(statsobj) + + +/* debugprint for the statsobj object */ +BEGINobjDebugPrint(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(statsobj) + dbgoprint((obj_t*) pThis, "statsobj object, currently no state info available\n"); +ENDobjDebugPrint(statsobj) + + +/* queryInterface function + */ +BEGINobjQueryInterface(statsobj) +CODESTARTobjQueryInterface(statsobj) + if(pIf->ifVersion != statsobjCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = statsobjConstruct; + pIf->ConstructFinalize = statsobjConstructFinalize; + pIf->Destruct = statsobjDestruct; + pIf->DebugPrint = statsobjDebugPrint; + pIf->SetName = setName; + pIf->GetStatsLine = getStatsLine; + pIf->GetAllStatsLines = getAllStatsLines; + pIf->AddCounter = addCounter; + pIf->EnableStats = enableStats; +finalize_it: +ENDobjQueryInterface(statsobj) + + +/* Initialize the statsobj class. Must be called as the very first method + * before anything else is called inside this class. + */ +BEGINAbstractObjClassInit(statsobj, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, statsobjDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, statsobjConstructFinalize); + + /* init other data items */ + pthread_mutex_init(&mutStats, NULL); + +ENDObjClassInit(statsobj) + +/* Exit the class. + */ +BEGINObjClassExit(statsobj, OBJ_IS_CORE_MODULE) /* class, version */ + /* release objects we no longer need */ + pthread_mutex_destroy(&mutStats); +ENDObjClassExit(statsobj) diff --git a/runtime/statsobj.h b/runtime/statsobj.h new file mode 100644 index 00000000..1ab02182 --- /dev/null +++ b/runtime/statsobj.h @@ -0,0 +1,124 @@ +/* The statsobj object. + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STATSOBJ_H +#define INCLUDED_STATSOBJ_H + +#include "atomic.h" + +/* The following data item is somewhat dirty, in that it does not follow + * our usual object calling conventions. However, much like with "Debug", we + * do this to gain speed. If we finally come to a platform that does not + * provide resolution of names for dynamically loaded modules, we need to find + * a work-around, but until then, we use the direct access. + * If set to 0, statistics are not gathered, otherwise they are. + */ +extern int GatherStats; + +/* our basic counter type -- need 32 bit on 32 bit platform. + * IMPORTANT: this type *MUST* be supported by atomic instructions! + */ +typedef uint64 intctr_t; + +/* counter types */ +typedef enum statsCtrType_e { + ctrType_IntCtr, + ctrType_Int +} statsCtrType_t; + + +/* helper entity, the counter */ +typedef struct ctr_s { + uchar *name; + statsCtrType_t ctrType; + union { + intctr_t *pIntCtr; + int *pInt; + } val; + struct ctr_s *next, *prev; +} ctr_t; + +/* the statsobj object */ +struct statsobj_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *name; + pthread_mutex_t mutCtr; /* to guard counter linked-list ops */ + ctr_t *ctrRoot; /* doubly-linked list of statsobj counters */ + ctr_t *ctrLast; + /* used to link ourselves together */ + statsobj_t *prev; + statsobj_t *next; +}; + + +/* interfaces */ +BEGINinterface(statsobj) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(statsobj); + rsRetVal (*Construct)(statsobj_t **ppThis); + rsRetVal (*ConstructFinalize)(statsobj_t *pThis); + rsRetVal (*Destruct)(statsobj_t **ppThis); + rsRetVal (*SetName)(statsobj_t *pThis, uchar *name); + rsRetVal (*GetStatsLine)(statsobj_t *pThis, cstr_t **ppcstr); + rsRetVal (*GetAllStatsLines)(rsRetVal(*cb)(void*, cstr_t*), void *usrptr); + rsRetVal (*AddCounter)(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, void *pCtr); + rsRetVal (*EnableStats)(void); +ENDinterface(statsobj) +#define statsobjCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(statsobj); + + +/* macros to handle stats counters + * These are to be used by "counter providers". Note that we MUST + * specify the mutex name, even though at first it looks like it + * could be automatically be generated via e.g. "mut##ctr". + * Unfortunately, this does not work if counter is e.g. "pThis->ctr". + * So we decided, for clarity, to always insist on specifying the mutex + * name (after all, it's just a few more keystrokes...). + */ +#define STATSCOUNTER_DEF(ctr, mut) \ + intctr_t ctr; \ + DEF_ATOMIC_HELPER_MUT(mut) + +#define STATSCOUNTER_INIT(ctr, mut) \ + INIT_ATOMIC_HELPER_MUT(mut); \ + ctr = 0; + +#define STATSCOUNTER_INC(ctr, mut) \ + if(GatherStats) \ + ATOMIC_INC(&ctr, mut); + +#define STATSCOUNTER_DEC(ctr, mut) \ + if(GatherStats) \ + ATOMIC_DEC(&ctr, mut); + +/* the next macro works only if the variable is already guarded + * by mutex (or the users risks a wrong result). It is assumed + * that there are not concurrent operations that modify the counter. + */ +#define STATSCOUNTER_SETMAX_NOMUT(ctr, newmax) \ + if(GatherStats && ((newmax) > (ctr))) \ + ctr = newmax; + +#endif /* #ifndef INCLUDED_STATSOBJ_H */ -- cgit From 2288bbb90fc6f51cebf5db81b05c6141e754ef25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Sep 2010 15:58:03 +0200 Subject: fixed compile problems on Solaris --- runtime/statsobj.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime') diff --git a/runtime/statsobj.h b/runtime/statsobj.h index 1ab02182..7447cded 100644 --- a/runtime/statsobj.h +++ b/runtime/statsobj.h @@ -99,7 +99,7 @@ PROTOTYPEObj(statsobj); */ #define STATSCOUNTER_DEF(ctr, mut) \ intctr_t ctr; \ - DEF_ATOMIC_HELPER_MUT(mut) + DEF_ATOMIC_HELPER_MUT(mut); #define STATSCOUNTER_INIT(ctr, mut) \ INIT_ATOMIC_HELPER_MUT(mut); \ @@ -107,7 +107,7 @@ PROTOTYPEObj(statsobj); #define STATSCOUNTER_INC(ctr, mut) \ if(GatherStats) \ - ATOMIC_INC(&ctr, mut); + ATOMIC_INC(&ctr, &mut); #define STATSCOUNTER_DEC(ctr, mut) \ if(GatherStats) \ -- cgit From 2a2cc3d26d56b834dcc143f2f4e3abbb6bd206c6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Sep 2010 12:20:59 +0200 Subject: minor: added version number to some error messages --- runtime/debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'runtime') diff --git a/runtime/debug.c b/runtime/debug.c index 64e251e5..5818dda9 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1310,7 +1310,7 @@ dbgGetRuntimeOptions(void) while(dbgGetRTOptNamVal(&pszOpts, &optname, &optval)) { if(!strcasecmp((char*)optname, "help")) { fprintf(stderr, - "rsyslogd runtime debug support - help requested, rsyslog terminates\n\n" + "rsyslogd " VERSION " runtime debug support - help requested, rsyslog terminates\n\n" "environment variables:\n" "addional logfile: export RSYSLOG_DEBUGFILE=\"/path/to/file\"\n" "to set: export RSYSLOG_DEBUG=\"cmd cmd cmd\"\n\n" @@ -1358,7 +1358,7 @@ dbgGetRuntimeOptions(void) bAbortTrace = 0; } else if(!strcasecmp((char*)optname, "filetrace")) { if(*optval == '\0') { - fprintf(stderr, "Error: logfile debug option requires filename, " + fprintf(stderr, "rsyslogd " VERSION " error: logfile debug option requires filename, " "e.g. \"logfile=debug.c\"\n"); exit(1); } else { @@ -1366,7 +1366,7 @@ dbgGetRuntimeOptions(void) dbgPrintNameAdd(optval, &printNameFileRoot); } } else { - fprintf(stderr, "Error: invalid debug option '%s', value '%s' - ignored\n", + fprintf(stderr, "rsyslogd " VERSION " error: invalid debug option '%s', value '%s' - ignored\n", optval, optname); } } -- cgit From c2d36c6894f7c36c2aed23c1e7ee9580c58659a7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 26 Sep 2010 18:57:05 +0200 Subject: patching SCM_CREDENTIALS pid into TAG of imuxsock read messages --- runtime/rsyslog.h | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 7b2655b0..c58c259b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -456,6 +456,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_EPOLL_CR_FAILED = -2173, /**< epoll_create() failed */ RS_RET_EPOLL_CTL_FAILED = -2174, /**< epoll_ctl() failed */ RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ + RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From d748c68c09ae7d4f8c22c4245e5509019c59511c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 11:32:48 +0200 Subject: added some generic hashtable code by Christopher Clark found at http://www.cl.cam.ac.uk/~cwc22/hashtable/ --- runtime/Makefile.am | 2 + runtime/hashtable/Makefile | 26 ++++ runtime/hashtable/hashtable.c | 275 ++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.h | 199 ++++++++++++++++++++++++ runtime/hashtable/hashtable_itr.c | 188 +++++++++++++++++++++++ runtime/hashtable/hashtable_itr.h | 112 ++++++++++++++ runtime/hashtable/hashtable_private.h | 85 +++++++++++ runtime/hashtable/hashtable_utility.c | 71 +++++++++ runtime/hashtable/hashtable_utility.h | 55 +++++++ runtime/hashtable/tester.c | 270 +++++++++++++++++++++++++++++++++ 10 files changed, 1283 insertions(+) create mode 100644 runtime/hashtable/Makefile create mode 100644 runtime/hashtable/hashtable.c create mode 100644 runtime/hashtable/hashtable.h create mode 100644 runtime/hashtable/hashtable_itr.c create mode 100644 runtime/hashtable/hashtable_itr.h create mode 100644 runtime/hashtable/hashtable_private.h create mode 100644 runtime/hashtable/hashtable_utility.c create mode 100644 runtime/hashtable/hashtable_utility.h create mode 100644 runtime/hashtable/tester.c (limited to 'runtime') diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 01b224e7..d657b6ca 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -95,6 +95,8 @@ librsyslog_la_SOURCES = \ ../parse.c \ ../parse.h \ \ + hashtable/hashtable.c \ + \ ../outchannel.c \ ../outchannel.h \ ../template.c \ diff --git a/runtime/hashtable/Makefile b/runtime/hashtable/Makefile new file mode 100644 index 00000000..3b7b5e9f --- /dev/null +++ b/runtime/hashtable/Makefile @@ -0,0 +1,26 @@ + +tester: hashtable.o tester.o hashtable_itr.o + gcc -g -Wall -O -lm -o tester hashtable.o hashtable_itr.o tester.o + +all: tester old_tester + +tester.o: tester.c + gcc -g -Wall -O -c tester.c -o tester.o + +old_tester: hashtable_powers.o tester.o hashtable_itr.o + gcc -g -Wall -O -o old_tester hashtable_powers.o hashtable_itr.o tester.o + +hashtable_powers.o: hashtable_powers.c + gcc -g -Wall -O -c hashtable_powers.c -o hashtable_powers.o + +hashtable.o: hashtable.c + gcc -g -Wall -O -c hashtable.c -o hashtable.o + +hashtable_itr.o: hashtable_itr.c + gcc -g -Wall -O -c hashtable_itr.c -o hashtable_itr.o + +tidy: + rm *.o + +clean: tidy + rm -f tester old_tester diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c new file mode 100644 index 00000000..a10e3bc6 --- /dev/null +++ b/runtime/hashtable/hashtable.c @@ -0,0 +1,275 @@ +/* Copyright (C) 2004 Christopher Clark */ +/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include +#include +#include +#include + +/* +Credit for primes table: Aaron Krowne + http://br.endernet.org/~akrowne/ + http://planetmath.org/encyclopedia/GoodHashTablePrimes.html +*/ +static const unsigned int primes[] = { +53, 97, 193, 389, +769, 1543, 3079, 6151, +12289, 24593, 49157, 98317, +196613, 393241, 786433, 1572869, +3145739, 6291469, 12582917, 25165843, +50331653, 100663319, 201326611, 402653189, +805306457, 1610612741 +}; +const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); +const float max_load_factor = 0.65; + +/*****************************************************************************/ +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashf) (void*), + int (*eqf) (void*,void*)) +{ + struct hashtable *h; + unsigned int pindex, size = primes[0]; + /* Check requested hashtable isn't too large */ + if (minsize > (1u << 30)) return NULL; + /* Enforce size as prime */ + for (pindex=0; pindex < prime_table_length; pindex++) { + if (primes[pindex] > minsize) { size = primes[pindex]; break; } + } + h = (struct hashtable *)malloc(sizeof(struct hashtable)); + if (NULL == h) return NULL; /*oom*/ + h->table = (struct entry **)malloc(sizeof(struct entry*) * size); + if (NULL == h->table) { free(h); return NULL; } /*oom*/ + memset(h->table, 0, size * sizeof(struct entry *)); + h->tablelength = size; + h->primeindex = pindex; + h->entrycount = 0; + h->hashfn = hashf; + h->eqfn = eqf; + h->loadlimit = (unsigned int) ceil(size * max_load_factor); + return h; +} + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k) +{ + /* Aim to protect against poor hash functions by adding logic here + * - logic taken from java 1.4 hashtable source */ + unsigned int i = h->hashfn(k); + i += ~(i << 9); + i ^= ((i >> 14) | (i << 18)); /* >>> */ + i += (i << 4); + i ^= ((i >> 10) | (i << 22)); /* >>> */ + return i; +} + +/*****************************************************************************/ +static int +hashtable_expand(struct hashtable *h) +{ + /* Double the size of the table to accomodate more entries */ + struct entry **newtable; + struct entry *e; + struct entry **pE; + unsigned int newsize, i, idx; + /* Check we're not hitting max capacity */ + if (h->primeindex == (prime_table_length - 1)) return 0; + newsize = primes[++(h->primeindex)]; + + newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); + if (NULL != newtable) + { + memset(newtable, 0, newsize * sizeof(struct entry *)); + /* This algorithm is not 'stable'. ie. it reverses the list + * when it transfers entries between the tables */ + for (i = 0; i < h->tablelength; i++) { + while (NULL != (e = h->table[i])) { + h->table[i] = e->next; + idx = indexFor(newsize,e->h); + e->next = newtable[idx]; + newtable[idx] = e; + } + } + free(h->table); + h->table = newtable; + } + /* Plan B: realloc instead */ + else + { + newtable = (struct entry **) + realloc(h->table, newsize * sizeof(struct entry *)); + if (NULL == newtable) { (h->primeindex)--; return 0; } + h->table = newtable; + memset(newtable[h->tablelength], 0, newsize - h->tablelength); + for (i = 0; i < h->tablelength; i++) { + for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { + idx = indexFor(newsize,e->h); + if (idx == i) + { + pE = &(e->next); + } + else + { + *pE = e->next; + e->next = newtable[idx]; + newtable[idx] = e; + } + } + } + } + h->tablelength = newsize; + h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + return -1; +} + +/*****************************************************************************/ +unsigned int +hashtable_count(struct hashtable *h) +{ + return h->entrycount; +} + +/*****************************************************************************/ +int +hashtable_insert(struct hashtable *h, void *k, void *v) +{ + /* This method allows duplicate keys - but they shouldn't be used */ + unsigned int idx; + struct entry *e; + if (++(h->entrycount) > h->loadlimit) + { + /* Ignore the return value. If expand fails, we should + * still try cramming just this value into the existing table + * -- we may not have memory for a larger table, but one more + * element may be ok. Next time we insert, we'll try expanding again.*/ + hashtable_expand(h); + } + e = (struct entry *)malloc(sizeof(struct entry)); + if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ + e->h = hash(h,k); + idx = indexFor(h->tablelength,e->h); + e->k = k; + e->v = v; + e->next = h->table[idx]; + h->table[idx] = e; + return -1; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_search(struct hashtable *h, void *k) +{ + struct entry *e; + unsigned int hashvalue, idx; + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hashvalue); + e = h->table[idx]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_remove(struct hashtable *h, void *k) +{ + /* TODO: consider compacting the table when the load factor drops enough, + * or provide a 'compact' method. */ + + struct entry *e; + struct entry **pE; + void *v; + unsigned int hashvalue, idx; + + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hash(h,k)); + pE = &(h->table[idx]); + e = *pE; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + *pE = e->next; + h->entrycount--; + v = e->v; + freekey(e->k); + free(e); + return v; + } + pE = &(e->next); + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +/* destroy */ +void +hashtable_destroy(struct hashtable *h, int free_values) +{ + unsigned int i; + struct entry *e, *f; + struct entry **table = h->table; + if (free_values) + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } + } + } + else + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f); } + } + } + free(h->table); + free(h); +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.h b/runtime/hashtable/hashtable.h new file mode 100644 index 00000000..b90781ab --- /dev/null +++ b/runtime/hashtable/hashtable.h @@ -0,0 +1,199 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_H__ +#define __HASHTABLE_CWC22_H__ + +struct hashtable; + +/* Example of use: + * + * struct hashtable *h; + * struct some_key *k; + * struct some_value *v; + * + * static unsigned int hash_from_key_fn( void *k ); + * static int keys_equal_fn ( void *key1, void *key2 ); + * + * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); + * k = (struct some_key *) malloc(sizeof(struct some_key)); + * v = (struct some_value *) malloc(sizeof(struct some_value)); + * + * (initialise k and v to suitable values) + * + * if (! hashtable_insert(h,k,v) ) + * { exit(-1); } + * + * if (NULL == (found = hashtable_search(h,k) )) + * { printf("not found!"); } + * + * if (NULL == (found = hashtable_remove(h,k) )) + * { printf("Not found\n"); } + * + */ + +/* Macros may be used to define type-safe(r) hashtable access functions, with + * methods specialized to take known key and value types as parameters. + * + * Example: + * + * Insert this at the start of your file: + * + * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); + * + * This defines the functions 'insert_some', 'search_some' and 'remove_some'. + * These operate just like hashtable_insert etc., with the same parameters, + * but their function signatures have 'struct some_key *' rather than + * 'void *', and hence can generate compile time errors if your program is + * supplying incorrect data as a key (and similarly for value). + * + * Note that the hash and key equality functions passed to create_hashtable + * still take 'void *' parameters instead of 'some key *'. This shouldn't be + * a difficult issue as they're only defined and passed once, and the other + * functions will ensure that only valid keys are supplied to them. + * + * The cost for this checking is increased code size and runtime overhead + * - if performance is important, it may be worth switching back to the + * unsafe methods once your program has been debugged with the safe methods. + * This just requires switching to some simple alternative defines - eg: + * #define insert_some hashtable_insert + * + */ + +/***************************************************************************** + * create_hashtable + + * @name create_hashtable + * @param minsize minimum initial size of hashtable + * @param hashfunction function for hashing keys + * @param key_eq_fn function for determining key equality + * @return newly created hashtable or NULL on failure + */ + +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashfunction) (void*), + int (*key_eq_fn) (void*,void*)); + +/***************************************************************************** + * hashtable_insert + + * @name hashtable_insert + * @param h the hashtable to insert into + * @param k the key - hashtable claims ownership and will free on removal + * @param v the value - does not claim ownership + * @return non-zero for successful insertion + * + * This function will cause the table to expand if the insertion would take + * the ratio of entries to table size over the maximum load factor. + * + * This function does not check for repeated insertions with a duplicate key. + * The value returned when using a duplicate key is undefined -- when + * the hashtable changes size, the order of retrieval of duplicate key + * entries is reversed. + * If in doubt, remove before insert. + */ + +int +hashtable_insert(struct hashtable *h, void *k, void *v); + +#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ +int fnname (struct hashtable *h, keytype *k, valuetype *v) \ +{ \ + return hashtable_insert(h,k,v); \ +} + +/***************************************************************************** + * hashtable_search + + * @name hashtable_search + * @param h the hashtable to search + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * +hashtable_search(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_search(h,k)); \ +} + +/***************************************************************************** + * hashtable_remove + + * @name hashtable_remove + * @param h the hashtable to remove the item from + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * /* returns value */ +hashtable_remove(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_remove(h,k)); \ +} + + +/***************************************************************************** + * hashtable_count + + * @name hashtable_count + * @param h the hashtable + * @return the number of items stored in the hashtable + */ +unsigned int +hashtable_count(struct hashtable *h); + + +/***************************************************************************** + * hashtable_destroy + + * @name hashtable_destroy + * @param h the hashtable + * @param free_values whether to call 'free' on the remaining values + */ + +void +hashtable_destroy(struct hashtable *h, int free_values); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_itr.c b/runtime/hashtable/hashtable_itr.c new file mode 100644 index 00000000..5dced841 --- /dev/null +++ b/runtime/hashtable/hashtable_itr.c @@ -0,0 +1,188 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_itr.h" +#include /* defines NULL */ + +/*****************************************************************************/ +/* hashtable_iterator - iterator constructor */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h) +{ + unsigned int i, tablelength; + struct hashtable_itr *itr = (struct hashtable_itr *) + malloc(sizeof(struct hashtable_itr)); + if (NULL == itr) return NULL; + itr->h = h; + itr->e = NULL; + itr->parent = NULL; + tablelength = h->tablelength; + itr->index = tablelength; + if (0 == h->entrycount) return itr; + + for (i = 0; i < tablelength; i++) + { + if (NULL != h->table[i]) + { + itr->e = h->table[i]; + itr->index = i; + break; + } + } + return itr; +} + +/*****************************************************************************/ +/* key - return the key of the (key,value) pair at the current position */ +/* value - return the value of the (key,value) pair at the current position */ + +void * +hashtable_iterator_key(struct hashtable_itr *i) +{ return i->e->k; } + +void * +hashtable_iterator_value(struct hashtable_itr *i) +{ return i->e->v; } + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr) +{ + unsigned int j,tablelength; + struct entry **table; + struct entry *next; + if (NULL == itr->e) return 0; /* stupidity check */ + + next = itr->e->next; + if (NULL != next) + { + itr->parent = itr->e; + itr->e = next; + return -1; + } + tablelength = itr->h->tablelength; + itr->parent = NULL; + if (tablelength <= (j = ++(itr->index))) + { + itr->e = NULL; + return 0; + } + table = itr->h->table; + while (NULL == (next = table[j])) + { + if (++j >= tablelength) + { + itr->index = tablelength; + itr->e = NULL; + return 0; + } + } + itr->index = j; + itr->e = next; + return -1; +} + +/*****************************************************************************/ +/* remove - remove the entry at the current iterator position + * and advance the iterator, if there is a successive + * element. + * If you want the value, read it before you remove: + * beware memory leaks if you don't. + * Returns zero if end of iteration. */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr) +{ + struct entry *remember_e, *remember_parent; + int ret; + + /* Do the removal */ + if (NULL == (itr->parent)) + { + /* element is head of a chain */ + itr->h->table[itr->index] = itr->e->next; + } else { + /* element is mid-chain */ + itr->parent->next = itr->e->next; + } + /* itr->e is now outside the hashtable */ + remember_e = itr->e; + itr->h->entrycount--; + freekey(remember_e->k); + + /* Advance the iterator, correcting the parent */ + remember_parent = itr->parent; + ret = hashtable_iterator_advance(itr); + if (itr->parent == remember_e) { itr->parent = remember_parent; } + free(remember_e); + return ret; +} + +/*****************************************************************************/ +int /* returns zero if not found */ +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k) +{ + struct entry *e, *parent; + unsigned int hashvalue, index; + + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + + e = h->table[index]; + parent = NULL; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + itr->index = index; + itr->e = e; + itr->parent = parent; + itr->h = h; + return -1; + } + parent = e; + e = e->next; + } + return 0; +} + + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_itr.h b/runtime/hashtable/hashtable_itr.h new file mode 100644 index 00000000..eea699a7 --- /dev/null +++ b/runtime/hashtable/hashtable_itr.h @@ -0,0 +1,112 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_ITR_CWC22__ +#define __HASHTABLE_ITR_CWC22__ +#include "hashtable.h" +#include "hashtable_private.h" /* needed to enable inlining */ + +/*****************************************************************************/ +/* This struct is only concrete here to allow the inlining of two of the + * accessor functions. */ +struct hashtable_itr +{ + struct hashtable *h; + struct entry *e; + struct entry *parent; + unsigned int index; +}; + + +/*****************************************************************************/ +/* hashtable_iterator + */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h); + +/*****************************************************************************/ +/* hashtable_iterator_key + * - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_key(struct hashtable_itr *i) +{ + return i->e->k; +} + +/*****************************************************************************/ +/* value - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_value(struct hashtable_itr *i) +{ + return i->e->v; +} + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* remove - remove current element and advance the iterator to the next element + * NB: if you need the value to free it, read it before + * removing. ie: beware memory leaks! + * returns zero if advanced to end of table */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* search - overwrite the supplied iterator, to point to the entry + * matching the supplied key. + h points to the hashtable to be searched. + * returns zero if not found. */ +int +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ +int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ +{ \ + return (hashtable_iterator_search(i,h,k)); \ +} + + + +#endif /* __HASHTABLE_ITR_CWC22__*/ + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h new file mode 100644 index 00000000..3e95f600 --- /dev/null +++ b/runtime/hashtable/hashtable_private.h @@ -0,0 +1,85 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_PRIVATE_CWC22_H__ +#define __HASHTABLE_PRIVATE_CWC22_H__ + +#include "hashtable.h" + +/*****************************************************************************/ +struct entry +{ + void *k, *v; + unsigned int h; + struct entry *next; +}; + +struct hashtable { + unsigned int tablelength; + struct entry **table; + unsigned int entrycount; + unsigned int loadlimit; + unsigned int primeindex; + unsigned int (*hashfn) (void *k); + int (*eqfn) (void *k1, void *k2); +}; + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k); + +/*****************************************************************************/ +/* indexFor */ +static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) { + return (hashvalue % tablelength); +}; + +/* Only works if tablelength == 2^N */ +/*static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) +{ + return (hashvalue & (tablelength - 1u)); +} +*/ + +/*****************************************************************************/ +#define freekey(X) free(X) +/*define freekey(X) ; */ + + +/*****************************************************************************/ + +#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_utility.c b/runtime/hashtable/hashtable_utility.c new file mode 100644 index 00000000..c3176709 --- /dev/null +++ b/runtime/hashtable/hashtable_utility.c @@ -0,0 +1,71 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_utility.h" +#include +#include +#include + +/*****************************************************************************/ +/* hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * */ +int +hashtable_change(struct hashtable *h, void *k, void *v) +{ + struct entry *e; + unsigned int hashvalue, index; + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + e = h->table[index]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + free(e->v); + e->v = v; + return -1; + } + e = e->next; + } + return 0; +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable_utility.h b/runtime/hashtable/hashtable_utility.h new file mode 100644 index 00000000..56a0ffd1 --- /dev/null +++ b/runtime/hashtable/hashtable_utility.h @@ -0,0 +1,55 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_UTILITY_H__ +#define __HASHTABLE_CWC22_UTILITY_H__ + +/***************************************************************************** + * hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * @name hashtable_change + * @param h the hashtable + * @param key + * @param value + * + */ +int +hashtable_change(struct hashtable *h, void *k, void *v); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/tester.c b/runtime/hashtable/tester.c new file mode 100644 index 00000000..4678ffa8 --- /dev/null +++ b/runtime/hashtable/tester.c @@ -0,0 +1,270 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_itr.h" +#include +#include +#include /* for memcmp */ + +static const int ITEM_COUNT = 4000; + +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; + +/*****************************************************************************/ +struct key +{ + uint32_t one_ip; uint32_t two_ip; uint16_t one_port; uint16_t two_port; +}; + +struct value +{ + char *id; +}; + +DEFINE_HASHTABLE_INSERT(insert_some, struct key, struct value); +DEFINE_HASHTABLE_SEARCH(search_some, struct key, struct value); +DEFINE_HASHTABLE_REMOVE(remove_some, struct key, struct value); +DEFINE_HASHTABLE_ITERATOR_SEARCH(search_itr_some, struct key); + + +/*****************************************************************************/ +static unsigned int +hashfromkey(void *ky) +{ + struct key *k = (struct key *)ky; + return (((k->one_ip << 17) | (k->one_ip >> 15)) ^ k->two_ip) + + (k->one_port * 17) + (k->two_port * 13 * 29); +} + +static int +equalkeys(void *k1, void *k2) +{ + return (0 == memcmp(k1,k2,sizeof(struct key))); +} + +/*****************************************************************************/ +int +main(int argc, char **argv) +{ + struct key *k, *kk; + struct value *v, *found; + struct hashtable *h; + struct hashtable_itr *itr; + int i; + + h = create_hashtable(16, hashfromkey, equalkeys); + if (NULL == h) exit(-1); /*oom*/ + + +/*****************************************************************************/ +/* Insertion */ + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) exit(-1); /*oom*/ + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable search */ + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = search_some(h,k))) { + printf("BUG: key not found\n"); + } + } + +/*****************************************************************************/ +/* Hashtable iteration */ + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(h); + i = 0; + if (hashtable_count(h) > 0) + { + do { + kk = hashtable_iterator_key(itr); + v = hashtable_iterator_value(itr); + /* here (kk,v) are a valid (key, value) pair */ + /* We could call 'hashtable_remove(h,kk)' - and this operation + * 'free's kk. However, the iterator is then broken. + * This is why hashtable_iterator_remove exists - see below. + */ + i++; + + } while (hashtable_iterator_advance(itr)); + } + printf("Iterated through %u entries.\n", i); + +/*****************************************************************************/ +/* Hashtable iterator search */ + + /* Try the search some method */ + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr,h,k)) { + printf("BUG: key not found searching with iterator"); + } + } + +/*****************************************************************************/ +/* Hashtable removal */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = remove_some(h,k))) { + printf("BUG: key not found for removal\n"); + } + } + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy and create */ + + hashtable_destroy(h, 1); + h = NULL; + free(k); + + h = create_hashtable(160, hashfromkey, equalkeys); + if (NULL == h) { + printf("out of memory allocating second hashtable\n"); + return 1; + } + +/*****************************************************************************/ +/* Hashtable insertion */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) + { + printf("out of memory inserting into second hashtable\n"); + return 1; + } + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable iterator search and iterator remove */ + + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = ITEM_COUNT - 1; i >= 0; i = i - 7) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr, h, k)) { + printf("BUG: key %u not found for search preremoval using iterator\n", i); + return 1; + } + if (0 == hashtable_iterator_remove(itr)) { + printf("BUG: key not found for removal using iterator\n"); + return 1; + } + } + free(itr); + +/*****************************************************************************/ +/* Hashtable iterator remove and advance */ + + for (itr = hashtable_iterator(h); + hashtable_iterator_remove(itr) != 0; ) { + ; + } + free(itr); + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy */ + + hashtable_destroy(h, 1); + free(k); + return 0; +} + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ -- cgit From 01a8807174d91a7936345d1172a87f98bbba61c4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 12:13:19 +0200 Subject: imuxsock: changed to per-pid ratelimiting --- runtime/hashtable/README | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 runtime/hashtable/README (limited to 'runtime') diff --git a/runtime/hashtable/README b/runtime/hashtable/README new file mode 100644 index 00000000..5cadde0c --- /dev/null +++ b/runtime/hashtable/README @@ -0,0 +1,11 @@ +This is the hashtable code provided by +Christopher Clark +available at http://www.cl.cam.ac.uk/~cwc22/hashtable/ + +It may be slightly modified. The plan is to streamline +the code based on our needs and "really" integrate it into +the rsyslog runtime library. For the time being, we use it from +inside this subdirectory. We do not need all files, but I thought +I keep them together in case we later need something else. + +rgerhards, 2010-09-28 -- cgit From 054d2ccdd6044f94823f8facbda935cb70646333 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 17:26:28 +0200 Subject: imuxsock: added per-socket hash tables/rate limiters & severity filter rate limiting now applies only to messages with a given severity or above. By default, emergency messages are NOT rate-limited. --- runtime/rsyslog.h | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index c58c259b..43203378 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -457,6 +457,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_EPOLL_CTL_FAILED = -2174, /**< epoll_ctl() failed */ RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ + RS_RET_RATE_LIMITED = -2177, /**< some messages discarded due to exceeding a rate limit */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From c7e36e6b7e3e9fb24dfd82e2cb7683bfee0c15ad Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Sep 2010 18:12:40 +0200 Subject: build system: fixed some minor issues --- runtime/Makefile.am | 2 + runtime/hashtable.h | 199 ++++++++++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.h | 199 ------------------------------------------ 3 files changed, 201 insertions(+), 199 deletions(-) create mode 100644 runtime/hashtable.h delete mode 100644 runtime/hashtable/hashtable.h (limited to 'runtime') diff --git a/runtime/Makefile.am b/runtime/Makefile.am index d657b6ca..5a0c4437 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -96,6 +96,8 @@ librsyslog_la_SOURCES = \ ../parse.h \ \ hashtable/hashtable.c \ + hashtable.h \ + hashtable/hashtable_private.h \ \ ../outchannel.c \ ../outchannel.h \ diff --git a/runtime/hashtable.h b/runtime/hashtable.h new file mode 100644 index 00000000..b90781ab --- /dev/null +++ b/runtime/hashtable.h @@ -0,0 +1,199 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_H__ +#define __HASHTABLE_CWC22_H__ + +struct hashtable; + +/* Example of use: + * + * struct hashtable *h; + * struct some_key *k; + * struct some_value *v; + * + * static unsigned int hash_from_key_fn( void *k ); + * static int keys_equal_fn ( void *key1, void *key2 ); + * + * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); + * k = (struct some_key *) malloc(sizeof(struct some_key)); + * v = (struct some_value *) malloc(sizeof(struct some_value)); + * + * (initialise k and v to suitable values) + * + * if (! hashtable_insert(h,k,v) ) + * { exit(-1); } + * + * if (NULL == (found = hashtable_search(h,k) )) + * { printf("not found!"); } + * + * if (NULL == (found = hashtable_remove(h,k) )) + * { printf("Not found\n"); } + * + */ + +/* Macros may be used to define type-safe(r) hashtable access functions, with + * methods specialized to take known key and value types as parameters. + * + * Example: + * + * Insert this at the start of your file: + * + * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); + * + * This defines the functions 'insert_some', 'search_some' and 'remove_some'. + * These operate just like hashtable_insert etc., with the same parameters, + * but their function signatures have 'struct some_key *' rather than + * 'void *', and hence can generate compile time errors if your program is + * supplying incorrect data as a key (and similarly for value). + * + * Note that the hash and key equality functions passed to create_hashtable + * still take 'void *' parameters instead of 'some key *'. This shouldn't be + * a difficult issue as they're only defined and passed once, and the other + * functions will ensure that only valid keys are supplied to them. + * + * The cost for this checking is increased code size and runtime overhead + * - if performance is important, it may be worth switching back to the + * unsafe methods once your program has been debugged with the safe methods. + * This just requires switching to some simple alternative defines - eg: + * #define insert_some hashtable_insert + * + */ + +/***************************************************************************** + * create_hashtable + + * @name create_hashtable + * @param minsize minimum initial size of hashtable + * @param hashfunction function for hashing keys + * @param key_eq_fn function for determining key equality + * @return newly created hashtable or NULL on failure + */ + +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashfunction) (void*), + int (*key_eq_fn) (void*,void*)); + +/***************************************************************************** + * hashtable_insert + + * @name hashtable_insert + * @param h the hashtable to insert into + * @param k the key - hashtable claims ownership and will free on removal + * @param v the value - does not claim ownership + * @return non-zero for successful insertion + * + * This function will cause the table to expand if the insertion would take + * the ratio of entries to table size over the maximum load factor. + * + * This function does not check for repeated insertions with a duplicate key. + * The value returned when using a duplicate key is undefined -- when + * the hashtable changes size, the order of retrieval of duplicate key + * entries is reversed. + * If in doubt, remove before insert. + */ + +int +hashtable_insert(struct hashtable *h, void *k, void *v); + +#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ +int fnname (struct hashtable *h, keytype *k, valuetype *v) \ +{ \ + return hashtable_insert(h,k,v); \ +} + +/***************************************************************************** + * hashtable_search + + * @name hashtable_search + * @param h the hashtable to search + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * +hashtable_search(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_search(h,k)); \ +} + +/***************************************************************************** + * hashtable_remove + + * @name hashtable_remove + * @param h the hashtable to remove the item from + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * /* returns value */ +hashtable_remove(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_remove(h,k)); \ +} + + +/***************************************************************************** + * hashtable_count + + * @name hashtable_count + * @param h the hashtable + * @return the number of items stored in the hashtable + */ +unsigned int +hashtable_count(struct hashtable *h); + + +/***************************************************************************** + * hashtable_destroy + + * @name hashtable_destroy + * @param h the hashtable + * @param free_values whether to call 'free' on the remaining values + */ + +void +hashtable_destroy(struct hashtable *h, int free_values); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.h b/runtime/hashtable/hashtable.h deleted file mode 100644 index b90781ab..00000000 --- a/runtime/hashtable/hashtable.h +++ /dev/null @@ -1,199 +0,0 @@ -/* Copyright (C) 2002 Christopher Clark */ - -#ifndef __HASHTABLE_CWC22_H__ -#define __HASHTABLE_CWC22_H__ - -struct hashtable; - -/* Example of use: - * - * struct hashtable *h; - * struct some_key *k; - * struct some_value *v; - * - * static unsigned int hash_from_key_fn( void *k ); - * static int keys_equal_fn ( void *key1, void *key2 ); - * - * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); - * k = (struct some_key *) malloc(sizeof(struct some_key)); - * v = (struct some_value *) malloc(sizeof(struct some_value)); - * - * (initialise k and v to suitable values) - * - * if (! hashtable_insert(h,k,v) ) - * { exit(-1); } - * - * if (NULL == (found = hashtable_search(h,k) )) - * { printf("not found!"); } - * - * if (NULL == (found = hashtable_remove(h,k) )) - * { printf("Not found\n"); } - * - */ - -/* Macros may be used to define type-safe(r) hashtable access functions, with - * methods specialized to take known key and value types as parameters. - * - * Example: - * - * Insert this at the start of your file: - * - * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); - * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); - * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); - * - * This defines the functions 'insert_some', 'search_some' and 'remove_some'. - * These operate just like hashtable_insert etc., with the same parameters, - * but their function signatures have 'struct some_key *' rather than - * 'void *', and hence can generate compile time errors if your program is - * supplying incorrect data as a key (and similarly for value). - * - * Note that the hash and key equality functions passed to create_hashtable - * still take 'void *' parameters instead of 'some key *'. This shouldn't be - * a difficult issue as they're only defined and passed once, and the other - * functions will ensure that only valid keys are supplied to them. - * - * The cost for this checking is increased code size and runtime overhead - * - if performance is important, it may be worth switching back to the - * unsafe methods once your program has been debugged with the safe methods. - * This just requires switching to some simple alternative defines - eg: - * #define insert_some hashtable_insert - * - */ - -/***************************************************************************** - * create_hashtable - - * @name create_hashtable - * @param minsize minimum initial size of hashtable - * @param hashfunction function for hashing keys - * @param key_eq_fn function for determining key equality - * @return newly created hashtable or NULL on failure - */ - -struct hashtable * -create_hashtable(unsigned int minsize, - unsigned int (*hashfunction) (void*), - int (*key_eq_fn) (void*,void*)); - -/***************************************************************************** - * hashtable_insert - - * @name hashtable_insert - * @param h the hashtable to insert into - * @param k the key - hashtable claims ownership and will free on removal - * @param v the value - does not claim ownership - * @return non-zero for successful insertion - * - * This function will cause the table to expand if the insertion would take - * the ratio of entries to table size over the maximum load factor. - * - * This function does not check for repeated insertions with a duplicate key. - * The value returned when using a duplicate key is undefined -- when - * the hashtable changes size, the order of retrieval of duplicate key - * entries is reversed. - * If in doubt, remove before insert. - */ - -int -hashtable_insert(struct hashtable *h, void *k, void *v); - -#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ -int fnname (struct hashtable *h, keytype *k, valuetype *v) \ -{ \ - return hashtable_insert(h,k,v); \ -} - -/***************************************************************************** - * hashtable_search - - * @name hashtable_search - * @param h the hashtable to search - * @param k the key to search for - does not claim ownership - * @return the value associated with the key, or NULL if none found - */ - -void * -hashtable_search(struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ -valuetype * fnname (struct hashtable *h, keytype *k) \ -{ \ - return (valuetype *) (hashtable_search(h,k)); \ -} - -/***************************************************************************** - * hashtable_remove - - * @name hashtable_remove - * @param h the hashtable to remove the item from - * @param k the key to search for - does not claim ownership - * @return the value associated with the key, or NULL if none found - */ - -void * /* returns value */ -hashtable_remove(struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ -valuetype * fnname (struct hashtable *h, keytype *k) \ -{ \ - return (valuetype *) (hashtable_remove(h,k)); \ -} - - -/***************************************************************************** - * hashtable_count - - * @name hashtable_count - * @param h the hashtable - * @return the number of items stored in the hashtable - */ -unsigned int -hashtable_count(struct hashtable *h); - - -/***************************************************************************** - * hashtable_destroy - - * @name hashtable_destroy - * @param h the hashtable - * @param free_values whether to call 'free' on the remaining values - */ - -void -hashtable_destroy(struct hashtable *h, int free_values); - -#endif /* __HASHTABLE_CWC22_H__ */ - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -- cgit From d8a1489f545179591abced679ba24831d85ca224 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 11:29:50 +0200 Subject: omhdfs: cleanup and lots of improvement now things look much better, also done some prep in order to support a file cache (we need this for multiple selectors writing to the same file). --- runtime/rsyslog.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'runtime') diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 43203378..7ccc9cb8 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -458,6 +458,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INTERNAL_ERROR = -2175, /**< rsyslogd internal error, unexpected code path reached */ RS_RET_ERR_CRE_AFUX = -2176, /**< error creating AF_UNIX socket (and binding it) */ RS_RET_RATE_LIMITED = -2177, /**< some messages discarded due to exceeding a rate limit */ + RS_RET_ERR_HDFS_WRITE = -2178, /**< error writing to HDFS */ + RS_RET_ERR_HDFS_OPEN = -2179, /**< error during hdfsOpen (e.g. file does not exist) */ + RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From 9696cdef34f5d033564138fb9d4afb87daa6b1be Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 12:32:01 +0200 Subject: omhdfs: files now kept inside a hashtable for use by multiple actions Note:compiles, but not yet tested --- runtime/hashtable.h | 4 +++- runtime/hashtable/hashtable.c | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/hashtable.h b/runtime/hashtable.h index b90781ab..0f980127 100644 --- a/runtime/hashtable.h +++ b/runtime/hashtable.h @@ -196,4 +196,6 @@ hashtable_destroy(struct hashtable *h, int free_values); * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ + */ +unsigned int hash_from_string(void *k) ; +int key_equals_string(void *key1, void *key2); diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c index a10e3bc6..e2a2b3f4 100644 --- a/runtime/hashtable/hashtable.c +++ b/runtime/hashtable/hashtable.c @@ -241,6 +241,33 @@ hashtable_destroy(struct hashtable *h, int free_values) free(h); } +/* some generic hash functions */ + +/* one provided by Aaaron Wiebe based on perl's hashng algorithm + * (so probably pretty generic). Not for excessively large strings! + */ +unsigned int +hash_from_string(void *k) +{ + int len; + char *rkey = (char*) k; + unsigned hashval = 1; + + len = (int) strlen(rkey); + while (len--) + hashval = hashval * 33 + *rkey++; + + return hashval; +} + + +int +key_equals_string(void *key1, void *key2) +{ + return strcmp(key1, key2); +} + + /* * Copyright (c) 2002, Christopher Clark * All rights reserved. -- cgit From 255895a58b3f2a54fecf971da700caf265b4e1f0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Oct 2010 13:59:48 +0000 Subject: omhdfs: more improvements finally this looks almost production ready for files where no directory path needs to be created --- runtime/hashtable.h | 3 ++- runtime/hashtable/hashtable.c | 17 ++++++++++++++--- runtime/hashtable/hashtable_private.h | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) (limited to 'runtime') diff --git a/runtime/hashtable.h b/runtime/hashtable.h index 0f980127..f777ad0b 100644 --- a/runtime/hashtable.h +++ b/runtime/hashtable.h @@ -68,13 +68,14 @@ struct hashtable; * @param minsize minimum initial size of hashtable * @param hashfunction function for hashing keys * @param key_eq_fn function for determining key equality + * @param dest destructor for value entries (NULL -> use free()) * @return newly created hashtable or NULL on failure */ struct hashtable * create_hashtable(unsigned int minsize, unsigned int (*hashfunction) (void*), - int (*key_eq_fn) (void*,void*)); + int (*key_eq_fn) (void*,void*), void (*dest) (void*)); /***************************************************************************** * hashtable_insert diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c index e2a2b3f4..41fc60fe 100644 --- a/runtime/hashtable/hashtable.c +++ b/runtime/hashtable/hashtable.c @@ -29,7 +29,7 @@ const float max_load_factor = 0.65; struct hashtable * create_hashtable(unsigned int minsize, unsigned int (*hashf) (void*), - int (*eqf) (void*,void*)) + int (*eqf) (void*,void*), void (*dest)(void*)) { struct hashtable *h; unsigned int pindex, size = primes[0]; @@ -49,6 +49,7 @@ create_hashtable(unsigned int minsize, h->entrycount = 0; h->hashfn = hashf; h->eqfn = eqf; + h->dest = dest; h->loadlimit = (unsigned int) ceil(size * max_load_factor); return h; } @@ -225,7 +226,16 @@ hashtable_destroy(struct hashtable *h, int free_values) { e = table[i]; while (NULL != e) - { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } + { + f = e; + e = e->next; + freekey(f->k); + if(h->dest == NULL) + free(f->v); + else + h->dest(f->v); + free(f); + } } } else @@ -264,7 +274,8 @@ hash_from_string(void *k) int key_equals_string(void *key1, void *key2) { - return strcmp(key1, key2); + /* we must return true IF the keys are equal! */ + return !strcmp(key1, key2); } diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h index 3e95f600..10b82da4 100644 --- a/runtime/hashtable/hashtable_private.h +++ b/runtime/hashtable/hashtable_private.h @@ -21,6 +21,7 @@ struct hashtable { unsigned int primeindex; unsigned int (*hashfn) (void *k); int (*eqfn) (void *k1, void *k2); + void (*dest) (void *v); /* destructor for values, if NULL use free() */ }; /*****************************************************************************/ -- cgit From e40cb595a2da943483124fae8f215a397add9fca Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Oct 2010 13:58:41 +0200 Subject: omhdfs: support for HUP added --- runtime/Makefile.am | 6 +- runtime/hashtable.c | 313 ++++++++++++++++++++++++++++++++++ runtime/hashtable/hashtable.c | 313 ---------------------------------- runtime/hashtable/hashtable_itr.c | 188 -------------------- runtime/hashtable/hashtable_itr.h | 112 ------------ runtime/hashtable/hashtable_private.h | 86 ---------- runtime/hashtable_itr.c | 190 +++++++++++++++++++++ runtime/hashtable_itr.h | 112 ++++++++++++ runtime/hashtable_private.h | 86 ++++++++++ 9 files changed, 705 insertions(+), 701 deletions(-) create mode 100644 runtime/hashtable.c delete mode 100644 runtime/hashtable/hashtable.c delete mode 100644 runtime/hashtable/hashtable_itr.c delete mode 100644 runtime/hashtable/hashtable_itr.h delete mode 100644 runtime/hashtable/hashtable_private.h create mode 100644 runtime/hashtable_itr.c create mode 100644 runtime/hashtable_itr.h create mode 100644 runtime/hashtable_private.h (limited to 'runtime') diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 5a0c4437..93817e75 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -95,9 +95,11 @@ librsyslog_la_SOURCES = \ ../parse.c \ ../parse.h \ \ - hashtable/hashtable.c \ + hashtable.c \ hashtable.h \ - hashtable/hashtable_private.h \ + hashtable_itr.c \ + hashtable_itr.h \ + hashtable_private.h \ \ ../outchannel.c \ ../outchannel.h \ diff --git a/runtime/hashtable.c b/runtime/hashtable.c new file mode 100644 index 00000000..41fc60fe --- /dev/null +++ b/runtime/hashtable.c @@ -0,0 +1,313 @@ +/* Copyright (C) 2004 Christopher Clark */ +/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include +#include +#include +#include + +/* +Credit for primes table: Aaron Krowne + http://br.endernet.org/~akrowne/ + http://planetmath.org/encyclopedia/GoodHashTablePrimes.html +*/ +static const unsigned int primes[] = { +53, 97, 193, 389, +769, 1543, 3079, 6151, +12289, 24593, 49157, 98317, +196613, 393241, 786433, 1572869, +3145739, 6291469, 12582917, 25165843, +50331653, 100663319, 201326611, 402653189, +805306457, 1610612741 +}; +const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); +const float max_load_factor = 0.65; + +/*****************************************************************************/ +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashf) (void*), + int (*eqf) (void*,void*), void (*dest)(void*)) +{ + struct hashtable *h; + unsigned int pindex, size = primes[0]; + /* Check requested hashtable isn't too large */ + if (minsize > (1u << 30)) return NULL; + /* Enforce size as prime */ + for (pindex=0; pindex < prime_table_length; pindex++) { + if (primes[pindex] > minsize) { size = primes[pindex]; break; } + } + h = (struct hashtable *)malloc(sizeof(struct hashtable)); + if (NULL == h) return NULL; /*oom*/ + h->table = (struct entry **)malloc(sizeof(struct entry*) * size); + if (NULL == h->table) { free(h); return NULL; } /*oom*/ + memset(h->table, 0, size * sizeof(struct entry *)); + h->tablelength = size; + h->primeindex = pindex; + h->entrycount = 0; + h->hashfn = hashf; + h->eqfn = eqf; + h->dest = dest; + h->loadlimit = (unsigned int) ceil(size * max_load_factor); + return h; +} + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k) +{ + /* Aim to protect against poor hash functions by adding logic here + * - logic taken from java 1.4 hashtable source */ + unsigned int i = h->hashfn(k); + i += ~(i << 9); + i ^= ((i >> 14) | (i << 18)); /* >>> */ + i += (i << 4); + i ^= ((i >> 10) | (i << 22)); /* >>> */ + return i; +} + +/*****************************************************************************/ +static int +hashtable_expand(struct hashtable *h) +{ + /* Double the size of the table to accomodate more entries */ + struct entry **newtable; + struct entry *e; + struct entry **pE; + unsigned int newsize, i, idx; + /* Check we're not hitting max capacity */ + if (h->primeindex == (prime_table_length - 1)) return 0; + newsize = primes[++(h->primeindex)]; + + newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); + if (NULL != newtable) + { + memset(newtable, 0, newsize * sizeof(struct entry *)); + /* This algorithm is not 'stable'. ie. it reverses the list + * when it transfers entries between the tables */ + for (i = 0; i < h->tablelength; i++) { + while (NULL != (e = h->table[i])) { + h->table[i] = e->next; + idx = indexFor(newsize,e->h); + e->next = newtable[idx]; + newtable[idx] = e; + } + } + free(h->table); + h->table = newtable; + } + /* Plan B: realloc instead */ + else + { + newtable = (struct entry **) + realloc(h->table, newsize * sizeof(struct entry *)); + if (NULL == newtable) { (h->primeindex)--; return 0; } + h->table = newtable; + memset(newtable[h->tablelength], 0, newsize - h->tablelength); + for (i = 0; i < h->tablelength; i++) { + for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { + idx = indexFor(newsize,e->h); + if (idx == i) + { + pE = &(e->next); + } + else + { + *pE = e->next; + e->next = newtable[idx]; + newtable[idx] = e; + } + } + } + } + h->tablelength = newsize; + h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + return -1; +} + +/*****************************************************************************/ +unsigned int +hashtable_count(struct hashtable *h) +{ + return h->entrycount; +} + +/*****************************************************************************/ +int +hashtable_insert(struct hashtable *h, void *k, void *v) +{ + /* This method allows duplicate keys - but they shouldn't be used */ + unsigned int idx; + struct entry *e; + if (++(h->entrycount) > h->loadlimit) + { + /* Ignore the return value. If expand fails, we should + * still try cramming just this value into the existing table + * -- we may not have memory for a larger table, but one more + * element may be ok. Next time we insert, we'll try expanding again.*/ + hashtable_expand(h); + } + e = (struct entry *)malloc(sizeof(struct entry)); + if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ + e->h = hash(h,k); + idx = indexFor(h->tablelength,e->h); + e->k = k; + e->v = v; + e->next = h->table[idx]; + h->table[idx] = e; + return -1; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_search(struct hashtable *h, void *k) +{ + struct entry *e; + unsigned int hashvalue, idx; + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hashvalue); + e = h->table[idx]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_remove(struct hashtable *h, void *k) +{ + /* TODO: consider compacting the table when the load factor drops enough, + * or provide a 'compact' method. */ + + struct entry *e; + struct entry **pE; + void *v; + unsigned int hashvalue, idx; + + hashvalue = hash(h,k); + idx = indexFor(h->tablelength,hash(h,k)); + pE = &(h->table[idx]); + e = *pE; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + *pE = e->next; + h->entrycount--; + v = e->v; + freekey(e->k); + free(e); + return v; + } + pE = &(e->next); + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +/* destroy */ +void +hashtable_destroy(struct hashtable *h, int free_values) +{ + unsigned int i; + struct entry *e, *f; + struct entry **table = h->table; + if (free_values) + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { + f = e; + e = e->next; + freekey(f->k); + if(h->dest == NULL) + free(f->v); + else + h->dest(f->v); + free(f); + } + } + } + else + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f); } + } + } + free(h->table); + free(h); +} + +/* some generic hash functions */ + +/* one provided by Aaaron Wiebe based on perl's hashng algorithm + * (so probably pretty generic). Not for excessively large strings! + */ +unsigned int +hash_from_string(void *k) +{ + int len; + char *rkey = (char*) k; + unsigned hashval = 1; + + len = (int) strlen(rkey); + while (len--) + hashval = hashval * 33 + *rkey++; + + return hashval; +} + + +int +key_equals_string(void *key1, void *key2) +{ + /* we must return true IF the keys are equal! */ + return !strcmp(key1, key2); +} + + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable/hashtable.c b/runtime/hashtable/hashtable.c deleted file mode 100644 index 41fc60fe..00000000 --- a/runtime/hashtable/hashtable.c +++ /dev/null @@ -1,313 +0,0 @@ -/* Copyright (C) 2004 Christopher Clark */ -/* taken from http://www.cl.cam.ac.uk/~cwc22/hashtable/ */ - -#include "hashtable.h" -#include "hashtable_private.h" -#include -#include -#include -#include - -/* -Credit for primes table: Aaron Krowne - http://br.endernet.org/~akrowne/ - http://planetmath.org/encyclopedia/GoodHashTablePrimes.html -*/ -static const unsigned int primes[] = { -53, 97, 193, 389, -769, 1543, 3079, 6151, -12289, 24593, 49157, 98317, -196613, 393241, 786433, 1572869, -3145739, 6291469, 12582917, 25165843, -50331653, 100663319, 201326611, 402653189, -805306457, 1610612741 -}; -const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); -const float max_load_factor = 0.65; - -/*****************************************************************************/ -struct hashtable * -create_hashtable(unsigned int minsize, - unsigned int (*hashf) (void*), - int (*eqf) (void*,void*), void (*dest)(void*)) -{ - struct hashtable *h; - unsigned int pindex, size = primes[0]; - /* Check requested hashtable isn't too large */ - if (minsize > (1u << 30)) return NULL; - /* Enforce size as prime */ - for (pindex=0; pindex < prime_table_length; pindex++) { - if (primes[pindex] > minsize) { size = primes[pindex]; break; } - } - h = (struct hashtable *)malloc(sizeof(struct hashtable)); - if (NULL == h) return NULL; /*oom*/ - h->table = (struct entry **)malloc(sizeof(struct entry*) * size); - if (NULL == h->table) { free(h); return NULL; } /*oom*/ - memset(h->table, 0, size * sizeof(struct entry *)); - h->tablelength = size; - h->primeindex = pindex; - h->entrycount = 0; - h->hashfn = hashf; - h->eqfn = eqf; - h->dest = dest; - h->loadlimit = (unsigned int) ceil(size * max_load_factor); - return h; -} - -/*****************************************************************************/ -unsigned int -hash(struct hashtable *h, void *k) -{ - /* Aim to protect against poor hash functions by adding logic here - * - logic taken from java 1.4 hashtable source */ - unsigned int i = h->hashfn(k); - i += ~(i << 9); - i ^= ((i >> 14) | (i << 18)); /* >>> */ - i += (i << 4); - i ^= ((i >> 10) | (i << 22)); /* >>> */ - return i; -} - -/*****************************************************************************/ -static int -hashtable_expand(struct hashtable *h) -{ - /* Double the size of the table to accomodate more entries */ - struct entry **newtable; - struct entry *e; - struct entry **pE; - unsigned int newsize, i, idx; - /* Check we're not hitting max capacity */ - if (h->primeindex == (prime_table_length - 1)) return 0; - newsize = primes[++(h->primeindex)]; - - newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); - if (NULL != newtable) - { - memset(newtable, 0, newsize * sizeof(struct entry *)); - /* This algorithm is not 'stable'. ie. it reverses the list - * when it transfers entries between the tables */ - for (i = 0; i < h->tablelength; i++) { - while (NULL != (e = h->table[i])) { - h->table[i] = e->next; - idx = indexFor(newsize,e->h); - e->next = newtable[idx]; - newtable[idx] = e; - } - } - free(h->table); - h->table = newtable; - } - /* Plan B: realloc instead */ - else - { - newtable = (struct entry **) - realloc(h->table, newsize * sizeof(struct entry *)); - if (NULL == newtable) { (h->primeindex)--; return 0; } - h->table = newtable; - memset(newtable[h->tablelength], 0, newsize - h->tablelength); - for (i = 0; i < h->tablelength; i++) { - for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { - idx = indexFor(newsize,e->h); - if (idx == i) - { - pE = &(e->next); - } - else - { - *pE = e->next; - e->next = newtable[idx]; - newtable[idx] = e; - } - } - } - } - h->tablelength = newsize; - h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); - return -1; -} - -/*****************************************************************************/ -unsigned int -hashtable_count(struct hashtable *h) -{ - return h->entrycount; -} - -/*****************************************************************************/ -int -hashtable_insert(struct hashtable *h, void *k, void *v) -{ - /* This method allows duplicate keys - but they shouldn't be used */ - unsigned int idx; - struct entry *e; - if (++(h->entrycount) > h->loadlimit) - { - /* Ignore the return value. If expand fails, we should - * still try cramming just this value into the existing table - * -- we may not have memory for a larger table, but one more - * element may be ok. Next time we insert, we'll try expanding again.*/ - hashtable_expand(h); - } - e = (struct entry *)malloc(sizeof(struct entry)); - if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ - e->h = hash(h,k); - idx = indexFor(h->tablelength,e->h); - e->k = k; - e->v = v; - e->next = h->table[idx]; - h->table[idx] = e; - return -1; -} - -/*****************************************************************************/ -void * /* returns value associated with key */ -hashtable_search(struct hashtable *h, void *k) -{ - struct entry *e; - unsigned int hashvalue, idx; - hashvalue = hash(h,k); - idx = indexFor(h->tablelength,hashvalue); - e = h->table[idx]; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; - e = e->next; - } - return NULL; -} - -/*****************************************************************************/ -void * /* returns value associated with key */ -hashtable_remove(struct hashtable *h, void *k) -{ - /* TODO: consider compacting the table when the load factor drops enough, - * or provide a 'compact' method. */ - - struct entry *e; - struct entry **pE; - void *v; - unsigned int hashvalue, idx; - - hashvalue = hash(h,k); - idx = indexFor(h->tablelength,hash(h,k)); - pE = &(h->table[idx]); - e = *pE; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) - { - *pE = e->next; - h->entrycount--; - v = e->v; - freekey(e->k); - free(e); - return v; - } - pE = &(e->next); - e = e->next; - } - return NULL; -} - -/*****************************************************************************/ -/* destroy */ -void -hashtable_destroy(struct hashtable *h, int free_values) -{ - unsigned int i; - struct entry *e, *f; - struct entry **table = h->table; - if (free_values) - { - for (i = 0; i < h->tablelength; i++) - { - e = table[i]; - while (NULL != e) - { - f = e; - e = e->next; - freekey(f->k); - if(h->dest == NULL) - free(f->v); - else - h->dest(f->v); - free(f); - } - } - } - else - { - for (i = 0; i < h->tablelength; i++) - { - e = table[i]; - while (NULL != e) - { f = e; e = e->next; freekey(f->k); free(f); } - } - } - free(h->table); - free(h); -} - -/* some generic hash functions */ - -/* one provided by Aaaron Wiebe based on perl's hashng algorithm - * (so probably pretty generic). Not for excessively large strings! - */ -unsigned int -hash_from_string(void *k) -{ - int len; - char *rkey = (char*) k; - unsigned hashval = 1; - - len = (int) strlen(rkey); - while (len--) - hashval = hashval * 33 + *rkey++; - - return hashval; -} - - -int -key_equals_string(void *key1, void *key2) -{ - /* we must return true IF the keys are equal! */ - return !strcmp(key1, key2); -} - - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_itr.c b/runtime/hashtable/hashtable_itr.c deleted file mode 100644 index 5dced841..00000000 --- a/runtime/hashtable/hashtable_itr.c +++ /dev/null @@ -1,188 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#include "hashtable.h" -#include "hashtable_private.h" -#include "hashtable_itr.h" -#include /* defines NULL */ - -/*****************************************************************************/ -/* hashtable_iterator - iterator constructor */ - -struct hashtable_itr * -hashtable_iterator(struct hashtable *h) -{ - unsigned int i, tablelength; - struct hashtable_itr *itr = (struct hashtable_itr *) - malloc(sizeof(struct hashtable_itr)); - if (NULL == itr) return NULL; - itr->h = h; - itr->e = NULL; - itr->parent = NULL; - tablelength = h->tablelength; - itr->index = tablelength; - if (0 == h->entrycount) return itr; - - for (i = 0; i < tablelength; i++) - { - if (NULL != h->table[i]) - { - itr->e = h->table[i]; - itr->index = i; - break; - } - } - return itr; -} - -/*****************************************************************************/ -/* key - return the key of the (key,value) pair at the current position */ -/* value - return the value of the (key,value) pair at the current position */ - -void * -hashtable_iterator_key(struct hashtable_itr *i) -{ return i->e->k; } - -void * -hashtable_iterator_value(struct hashtable_itr *i) -{ return i->e->v; } - -/*****************************************************************************/ -/* advance - advance the iterator to the next element - * returns zero if advanced to end of table */ - -int -hashtable_iterator_advance(struct hashtable_itr *itr) -{ - unsigned int j,tablelength; - struct entry **table; - struct entry *next; - if (NULL == itr->e) return 0; /* stupidity check */ - - next = itr->e->next; - if (NULL != next) - { - itr->parent = itr->e; - itr->e = next; - return -1; - } - tablelength = itr->h->tablelength; - itr->parent = NULL; - if (tablelength <= (j = ++(itr->index))) - { - itr->e = NULL; - return 0; - } - table = itr->h->table; - while (NULL == (next = table[j])) - { - if (++j >= tablelength) - { - itr->index = tablelength; - itr->e = NULL; - return 0; - } - } - itr->index = j; - itr->e = next; - return -1; -} - -/*****************************************************************************/ -/* remove - remove the entry at the current iterator position - * and advance the iterator, if there is a successive - * element. - * If you want the value, read it before you remove: - * beware memory leaks if you don't. - * Returns zero if end of iteration. */ - -int -hashtable_iterator_remove(struct hashtable_itr *itr) -{ - struct entry *remember_e, *remember_parent; - int ret; - - /* Do the removal */ - if (NULL == (itr->parent)) - { - /* element is head of a chain */ - itr->h->table[itr->index] = itr->e->next; - } else { - /* element is mid-chain */ - itr->parent->next = itr->e->next; - } - /* itr->e is now outside the hashtable */ - remember_e = itr->e; - itr->h->entrycount--; - freekey(remember_e->k); - - /* Advance the iterator, correcting the parent */ - remember_parent = itr->parent; - ret = hashtable_iterator_advance(itr); - if (itr->parent == remember_e) { itr->parent = remember_parent; } - free(remember_e); - return ret; -} - -/*****************************************************************************/ -int /* returns zero if not found */ -hashtable_iterator_search(struct hashtable_itr *itr, - struct hashtable *h, void *k) -{ - struct entry *e, *parent; - unsigned int hashvalue, index; - - hashvalue = hash(h,k); - index = indexFor(h->tablelength,hashvalue); - - e = h->table[index]; - parent = NULL; - while (NULL != e) - { - /* Check hash value to short circuit heavier comparison */ - if ((hashvalue == e->h) && (h->eqfn(k, e->k))) - { - itr->index = index; - itr->e = e; - itr->parent = parent; - itr->h = h; - return -1; - } - parent = e; - e = e->next; - } - return 0; -} - - -/* - * Copyright (c) 2002, 2004, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_itr.h b/runtime/hashtable/hashtable_itr.h deleted file mode 100644 index eea699a7..00000000 --- a/runtime/hashtable/hashtable_itr.h +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#ifndef __HASHTABLE_ITR_CWC22__ -#define __HASHTABLE_ITR_CWC22__ -#include "hashtable.h" -#include "hashtable_private.h" /* needed to enable inlining */ - -/*****************************************************************************/ -/* This struct is only concrete here to allow the inlining of two of the - * accessor functions. */ -struct hashtable_itr -{ - struct hashtable *h; - struct entry *e; - struct entry *parent; - unsigned int index; -}; - - -/*****************************************************************************/ -/* hashtable_iterator - */ - -struct hashtable_itr * -hashtable_iterator(struct hashtable *h); - -/*****************************************************************************/ -/* hashtable_iterator_key - * - return the value of the (key,value) pair at the current position */ - -extern inline void * -hashtable_iterator_key(struct hashtable_itr *i) -{ - return i->e->k; -} - -/*****************************************************************************/ -/* value - return the value of the (key,value) pair at the current position */ - -extern inline void * -hashtable_iterator_value(struct hashtable_itr *i) -{ - return i->e->v; -} - -/*****************************************************************************/ -/* advance - advance the iterator to the next element - * returns zero if advanced to end of table */ - -int -hashtable_iterator_advance(struct hashtable_itr *itr); - -/*****************************************************************************/ -/* remove - remove current element and advance the iterator to the next element - * NB: if you need the value to free it, read it before - * removing. ie: beware memory leaks! - * returns zero if advanced to end of table */ - -int -hashtable_iterator_remove(struct hashtable_itr *itr); - -/*****************************************************************************/ -/* search - overwrite the supplied iterator, to point to the entry - * matching the supplied key. - h points to the hashtable to be searched. - * returns zero if not found. */ -int -hashtable_iterator_search(struct hashtable_itr *itr, - struct hashtable *h, void *k); - -#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ -int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ -{ \ - return (hashtable_iterator_search(i,h,k)); \ -} - - - -#endif /* __HASHTABLE_ITR_CWC22__*/ - -/* - * Copyright (c) 2002, 2004, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable/hashtable_private.h b/runtime/hashtable/hashtable_private.h deleted file mode 100644 index 10b82da4..00000000 --- a/runtime/hashtable/hashtable_private.h +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright (C) 2002, 2004 Christopher Clark */ - -#ifndef __HASHTABLE_PRIVATE_CWC22_H__ -#define __HASHTABLE_PRIVATE_CWC22_H__ - -#include "hashtable.h" - -/*****************************************************************************/ -struct entry -{ - void *k, *v; - unsigned int h; - struct entry *next; -}; - -struct hashtable { - unsigned int tablelength; - struct entry **table; - unsigned int entrycount; - unsigned int loadlimit; - unsigned int primeindex; - unsigned int (*hashfn) (void *k); - int (*eqfn) (void *k1, void *k2); - void (*dest) (void *v); /* destructor for values, if NULL use free() */ -}; - -/*****************************************************************************/ -unsigned int -hash(struct hashtable *h, void *k); - -/*****************************************************************************/ -/* indexFor */ -static inline unsigned int -indexFor(unsigned int tablelength, unsigned int hashvalue) { - return (hashvalue % tablelength); -}; - -/* Only works if tablelength == 2^N */ -/*static inline unsigned int -indexFor(unsigned int tablelength, unsigned int hashvalue) -{ - return (hashvalue & (tablelength - 1u)); -} -*/ - -/*****************************************************************************/ -#define freekey(X) free(X) -/*define freekey(X) ; */ - - -/*****************************************************************************/ - -#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ - -/* - * Copyright (c) 2002, Christopher Clark - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of the original author; nor the names of any contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ diff --git a/runtime/hashtable_itr.c b/runtime/hashtable_itr.c new file mode 100644 index 00000000..967287f1 --- /dev/null +++ b/runtime/hashtable_itr.c @@ -0,0 +1,190 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_itr.h" +#include /* defines NULL */ + +/*****************************************************************************/ +/* hashtable_iterator - iterator constructor */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h) +{ + unsigned int i, tablelength; + struct hashtable_itr *itr = (struct hashtable_itr *) + malloc(sizeof(struct hashtable_itr)); + if (NULL == itr) return NULL; + itr->h = h; + itr->e = NULL; + itr->parent = NULL; + tablelength = h->tablelength; + itr->index = tablelength; + if (0 == h->entrycount) return itr; + + for (i = 0; i < tablelength; i++) + { + if (NULL != h->table[i]) + { + itr->e = h->table[i]; + itr->index = i; + break; + } + } + return itr; +} + +/*****************************************************************************/ +/* key - return the key of the (key,value) pair at the current position */ +/* value - return the value of the (key,value) pair at the current position */ + +#if 0 /* these are now inline functions! */ +void * +hashtable_iterator_key(struct hashtable_itr *i) +{ return i->e->k; } + +void * +hashtable_iterator_value(struct hashtable_itr *i) +{ return i->e->v; } +#endif + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr) +{ + unsigned int j,tablelength; + struct entry **table; + struct entry *next; + if (NULL == itr->e) return 0; /* stupidity check */ + + next = itr->e->next; + if (NULL != next) + { + itr->parent = itr->e; + itr->e = next; + return -1; + } + tablelength = itr->h->tablelength; + itr->parent = NULL; + if (tablelength <= (j = ++(itr->index))) + { + itr->e = NULL; + return 0; + } + table = itr->h->table; + while (NULL == (next = table[j])) + { + if (++j >= tablelength) + { + itr->index = tablelength; + itr->e = NULL; + return 0; + } + } + itr->index = j; + itr->e = next; + return -1; +} + +/*****************************************************************************/ +/* remove - remove the entry at the current iterator position + * and advance the iterator, if there is a successive + * element. + * If you want the value, read it before you remove: + * beware memory leaks if you don't. + * Returns zero if end of iteration. */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr) +{ + struct entry *remember_e, *remember_parent; + int ret; + + /* Do the removal */ + if (NULL == (itr->parent)) + { + /* element is head of a chain */ + itr->h->table[itr->index] = itr->e->next; + } else { + /* element is mid-chain */ + itr->parent->next = itr->e->next; + } + /* itr->e is now outside the hashtable */ + remember_e = itr->e; + itr->h->entrycount--; + freekey(remember_e->k); + + /* Advance the iterator, correcting the parent */ + remember_parent = itr->parent; + ret = hashtable_iterator_advance(itr); + if (itr->parent == remember_e) { itr->parent = remember_parent; } + free(remember_e); + return ret; +} + +/*****************************************************************************/ +int /* returns zero if not found */ +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k) +{ + struct entry *e, *parent; + unsigned int hashvalue, index; + + hashvalue = hash(h,k); + index = indexFor(h->tablelength,hashvalue); + + e = h->table[index]; + parent = NULL; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + itr->index = index; + itr->e = e; + itr->parent = parent; + itr->h = h; + return -1; + } + parent = e; + e = e->next; + } + return 0; +} + + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable_itr.h b/runtime/hashtable_itr.h new file mode 100644 index 00000000..1c206b6e --- /dev/null +++ b/runtime/hashtable_itr.h @@ -0,0 +1,112 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_ITR_CWC22__ +#define __HASHTABLE_ITR_CWC22__ +#include "hashtable.h" +#include "hashtable_private.h" /* needed to enable inlining */ + +/*****************************************************************************/ +/* This struct is only concrete here to allow the inlining of two of the + * accessor functions. */ +struct hashtable_itr +{ + struct hashtable *h; + struct entry *e; + struct entry *parent; + unsigned int index; +}; + + +/*****************************************************************************/ +/* hashtable_iterator + */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h); + +/*****************************************************************************/ +/* hashtable_iterator_key + * - return the value of the (key,value) pair at the current position */ + +static inline void * +hashtable_iterator_key(struct hashtable_itr *i) +{ + return i->e->k; +} + +/*****************************************************************************/ +/* value - return the value of the (key,value) pair at the current position */ + +static inline void * +hashtable_iterator_value(struct hashtable_itr *i) +{ + return i->e->v; +} + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* remove - remove current element and advance the iterator to the next element + * NB: if you need the value to free it, read it before + * removing. ie: beware memory leaks! + * returns zero if advanced to end of table */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* search - overwrite the supplied iterator, to point to the entry + * matching the supplied key. + h points to the hashtable to be searched. + * returns zero if not found. */ +int +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ +int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ +{ \ + return (hashtable_iterator_search(i,h,k)); \ +} + + + +#endif /* __HASHTABLE_ITR_CWC22__*/ + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/runtime/hashtable_private.h b/runtime/hashtable_private.h new file mode 100644 index 00000000..10b82da4 --- /dev/null +++ b/runtime/hashtable_private.h @@ -0,0 +1,86 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_PRIVATE_CWC22_H__ +#define __HASHTABLE_PRIVATE_CWC22_H__ + +#include "hashtable.h" + +/*****************************************************************************/ +struct entry +{ + void *k, *v; + unsigned int h; + struct entry *next; +}; + +struct hashtable { + unsigned int tablelength; + struct entry **table; + unsigned int entrycount; + unsigned int loadlimit; + unsigned int primeindex; + unsigned int (*hashfn) (void *k); + int (*eqfn) (void *k1, void *k2); + void (*dest) (void *v); /* destructor for values, if NULL use free() */ +}; + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k); + +/*****************************************************************************/ +/* indexFor */ +static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) { + return (hashvalue % tablelength); +}; + +/* Only works if tablelength == 2^N */ +/*static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) +{ + return (hashvalue & (tablelength - 1u)); +} +*/ + +/*****************************************************************************/ +#define freekey(X) free(X) +/*define freekey(X) ; */ + + +/*****************************************************************************/ + +#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ -- cgit From 8c3d40b798e3ed68fb03629d87b55140cb6bc044 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 Oct 2010 17:11:00 +0200 Subject: removed need for math library by doing math a little bit more optimal in hash table code. Also reduced memory requirement for imuxsock hash tables (expected number of connections was set too high -- table can be extended dynamically). --- runtime/hashtable.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'runtime') diff --git a/runtime/hashtable.c b/runtime/hashtable.c index 41fc60fe..a01fa7d9 100644 --- a/runtime/hashtable.c +++ b/runtime/hashtable.c @@ -23,7 +23,17 @@ static const unsigned int primes[] = { 805306457, 1610612741 }; const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); -const float max_load_factor = 0.65; + +#define MAX_LOAD_FACTOR 65 /* to get real factor, divide by 100! */ + +/* compute max load. We use a constant factor of 0.65, but do + * everything times 100, so that we do not need floats. + */ +static inline unsigned +getLoadLimit(unsigned size) +{ + return (unsigned int) ((unsigned long long) size * MAX_LOAD_FACTOR) / 100; +} /*****************************************************************************/ struct hashtable * @@ -50,7 +60,7 @@ create_hashtable(unsigned int minsize, h->hashfn = hashf; h->eqfn = eqf; h->dest = dest; - h->loadlimit = (unsigned int) ceil(size * max_load_factor); + h->loadlimit = getLoadLimit(size); return h; } @@ -123,7 +133,7 @@ hashtable_expand(struct hashtable *h) } } h->tablelength = newsize; - h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + h->loadlimit = getLoadLimit(newsize); return -1; } -- cgit From ed9fd3541f07929008a64f5b6ea3bcce70e62c25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 15 Oct 2010 08:23:29 +0200 Subject: imfile: bug fixes either one or two bugs fixed ;) Definitely a problem where no state file is written when working with relative pathes. Also, some problems with offsets should be fixed for very large files. However, I could not yet experimentally show the issue so it probably needs more verification. --- runtime/stream.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'runtime') diff --git a/runtime/stream.c b/runtime/stream.c index b4295762..7f965029 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1473,7 +1473,7 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) { DEFiRet; int i; - long l; + int64 ll; ISOBJ_TYPE_assert(pThis, strm); ISOBJ_TYPE_assert(pStrm, strm); @@ -1495,8 +1495,8 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) i = pThis->tOpenMode; objSerializeSCALAR_VAR(pStrm, tOpenMode, INT, i); - l = (long) pThis->iCurrOffs; - objSerializeSCALAR_VAR(pStrm, iCurrOffs, LONG, l); + ll = pThis->iCurrOffs; + objSerializeSCALAR_VAR(pStrm, iCurrOffs, INT64, ll); CHKiRet(obj.EndSerialize(pStrm)); -- cgit From b9151f6a1708b2fb7e9518e73fcf42d86b0364a9 Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Thu, 25 Nov 2010 15:44:49 +0100 Subject: bugfix: atomic increment for msg object may not work correct on all platforms. Signed-off-by: Rainer Gerhards --- runtime/msg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/msg.h b/runtime/msg.h index cda206fc..a6923d52 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -60,9 +60,9 @@ struct msg { flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ pthread_mutex_t mut; + int iRefCount; /* reference counter (0 = unused) */ bool bDoLock; /* use the mutex? */ bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ - short iRefCount; /* reference counter (0 = unused) */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, * it is available when we have a forwarder (e.g. rfc3195d) using local -- cgit From f8769ca19da2eff26dd80ca3b6217d078db1a59f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Dec 2010 18:13:40 +0100 Subject: bugfix: fixed build problems on some platforms namely those that have 32bit atomic operations but not 64 bit ones --- runtime/atomic.h | 27 +++++++++++++++++++++++++++ runtime/statsobj.h | 8 ++++---- 2 files changed, 31 insertions(+), 4 deletions(-) (limited to 'runtime') diff --git a/runtime/atomic.h b/runtime/atomic.h index 790cb1c6..c022035c 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -208,4 +208,31 @@ #endif +/* we need to handle 64bit atomics seperately as some platforms have + * 32 bit atomics, but not 64 biot ones... -- rgerhards, 2010-12-01 + */ +#ifdef HAVE_ATOMIC_BUILTINS_64BIT +# define ATOMIC_INC_uint64(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) +# define ATOMIC_DEC_unit64(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) + +# define DEF_ATOMIC_HELPER_MUT64(x) +# define INIT_ATOMIC_HELPER_MUT64(x) +# define DESTROY_ATOMIC_HELPER_MUT64(x) +#else +# define ATOMIC_INC_uint64(data, phlpmut) { \ + pthread_mutex_lock(phlpmut); \ + ++(*(data)); \ + pthread_mutex_unlock(phlpmut); \ + } +# define ATOMIC_DEC_uint64(data, phlpmut) { \ + pthread_mutex_lock(phlpmut); \ + --(*(data)); \ + pthread_mutex_unlock(phlpmut); \ + } + +# define DEF_ATOMIC_HELPER_MUT64(x) pthread_mutex_t x +# define INIT_ATOMIC_HELPER_MUT64(x) pthread_mutex_init(&(x), NULL) +# define DESTROY_ATOMIC_HELPER_MUT64(x) pthread_mutex_destroy(&(x)) +#endif /* #ifdef HAVE_ATOMIC_BUILTINS_64BIT */ + #endif /* #ifndef INCLUDED_ATOMIC_H */ diff --git a/runtime/statsobj.h b/runtime/statsobj.h index 7447cded..44c26bea 100644 --- a/runtime/statsobj.h +++ b/runtime/statsobj.h @@ -99,19 +99,19 @@ PROTOTYPEObj(statsobj); */ #define STATSCOUNTER_DEF(ctr, mut) \ intctr_t ctr; \ - DEF_ATOMIC_HELPER_MUT(mut); + DEF_ATOMIC_HELPER_MUT64(mut); #define STATSCOUNTER_INIT(ctr, mut) \ - INIT_ATOMIC_HELPER_MUT(mut); \ + INIT_ATOMIC_HELPER_MUT64(mut); \ ctr = 0; #define STATSCOUNTER_INC(ctr, mut) \ if(GatherStats) \ - ATOMIC_INC(&ctr, &mut); + ATOMIC_INC_uint64(&ctr, &mut); #define STATSCOUNTER_DEC(ctr, mut) \ if(GatherStats) \ - ATOMIC_DEC(&ctr, mut); + ATOMIC_DEC_uint64(&ctr, mut); /* the next macro works only if the variable is already guarded * by mutex (or the users risks a wrong result). It is assumed -- cgit From 1bfb97e576d779a709e1e2979eef4697b9b0cd16 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Dec 2010 07:28:04 +0100 Subject: bugfix: one type of 64bit atomics was enabled when 32bit atomics were supported also cleaned up some minor things --- runtime/atomic.h | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) (limited to 'runtime') diff --git a/runtime/atomic.h b/runtime/atomic.h index c022035c..3e2c0d18 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -46,7 +46,6 @@ # define ATOMIC_INC(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) # define ATOMIC_INC_AND_FETCH_int(data, phlpmut) __sync_fetch_and_add(data, 1) # define ATOMIC_INC_AND_FETCH_unsigned(data, phlpmut) __sync_fetch_and_add(data, 1) -# define ATOMIC_INC_AND_FETCH_uint64(data, phlpmut) __sync_fetch_and_add(data, 1) # define ATOMIC_DEC(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) # define ATOMIC_DEC_AND_FETCH(data, phlpmut) __sync_sub_and_fetch(data, 1) # define ATOMIC_FETCH_32BIT(data, phlpmut) ((unsigned) __sync_fetch_and_and(data, 0xffffffff)) @@ -160,15 +159,6 @@ return(val); } - static inline unsigned - ATOMIC_INC_AND_FETCH_uint64(uint64 *data, pthread_mutex_t *phlpmut) { - uint64 val; - pthread_mutex_lock(phlpmut); - val = ++(*data); - pthread_mutex_unlock(phlpmut); - return(val); - } - static inline int ATOMIC_DEC_AND_FETCH(int *data, pthread_mutex_t *phlpmut) { int val; @@ -193,13 +183,6 @@ (*data) -= val; pthread_mutex_unlock(phlpmut); } -#if 0 -# warning "atomic builtins not available, using nul operations - rsyslogd will probably be racy!" -# define ATOMIC_INC_AND_FETCH_int(data) (++(data)) -# define ATOMIC_INC_AND_FETCH_unsigned(data) (++(data)) -# define ATOMIC_INC_AND_FETCH_uint64(data) (++(data)) -# define ATOMIC_STORE_1_TO_32BIT(data) (data) = 1 // TODO: del -#endif # define DEF_ATOMIC_HELPER_MUT(x) pthread_mutex_t x # define INIT_ATOMIC_HELPER_MUT(x) pthread_mutex_init(&(x), NULL) # define DESTROY_ATOMIC_HELPER_MUT(x) pthread_mutex_destroy(&(x)) @@ -214,6 +197,7 @@ #ifdef HAVE_ATOMIC_BUILTINS_64BIT # define ATOMIC_INC_uint64(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) # define ATOMIC_DEC_unit64(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) +# define ATOMIC_INC_AND_FETCH_uint64(data, phlpmut) __sync_fetch_and_add(data, 1) # define DEF_ATOMIC_HELPER_MUT64(x) # define INIT_ATOMIC_HELPER_MUT64(x) @@ -230,6 +214,15 @@ pthread_mutex_unlock(phlpmut); \ } + static inline unsigned + ATOMIC_INC_AND_FETCH_uint64(uint64 *data, pthread_mutex_t *phlpmut) { + uint64 val; + pthread_mutex_lock(phlpmut); + val = ++(*data); + pthread_mutex_unlock(phlpmut); + return(val); + } + # define DEF_ATOMIC_HELPER_MUT64(x) pthread_mutex_t x # define INIT_ATOMIC_HELPER_MUT64(x) pthread_mutex_init(&(x), NULL) # define DESTROY_ATOMIC_HELPER_MUT64(x) pthread_mutex_destroy(&(x)) -- cgit From 699d0d933ab64941d40df17c69b2c377231924cf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Dec 2010 15:29:20 +0100 Subject: added $LocalHostName config directive & some bugfixing - added $LocalHostName config directive - bugfix: local hostname was pulled too-early, so that some config directives (namely FQDN settings) did not have any effect --- runtime/cfsysline.c | 3 --- runtime/glbl.c | 27 +++++++++++++++++++-------- 2 files changed, 19 insertions(+), 11 deletions(-) (limited to 'runtime') diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index 5ab73184..037e9f84 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -953,8 +953,6 @@ finalize_it: */ void dbgPrintCfSysLineHandlers(void) { - DEFiRet; - cslCmd_t *pCmd; cslCmdHdlr_t *pCmdHdlr; linkedListCookie_t llCookieCmd; @@ -976,7 +974,6 @@ void dbgPrintCfSysLineHandlers(void) } } dbgprintf("\n"); - ENDfunc } diff --git a/runtime/glbl.c b/runtime/glbl.c index 58558ed2..b396ed7c 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -64,6 +64,7 @@ static int option_DisallowWarning = 1; /* complain if message from disallowed se static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ static prop_t *propLocalHostName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup */ +static uchar *LocalHostNameOverride = NULL;/* user-overridden hostname - read-only after startup */ static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalDomain; /* our local domain name - read-only after startup */ static char **StripDomains = NULL;/* these domains may be stripped before writing logs - r/o after s.u., never touched by init */ @@ -153,17 +154,21 @@ GenerateLocalHostNameProperty(void) uchar *pszName; if(propLocalHostName != NULL) - prop.Destruct(&propLocalHostName); CHKiRet(prop.Construct(&propLocalHostName)); - if(LocalHostName == NULL) - pszName = (uchar*) "[localhost]"; - else { - if(GetPreserveFQDN() == 1) - pszName = LocalFQDNName; - else - pszName = LocalHostName; + if(LocalHostNameOverride == NULL) { + if(LocalHostName == NULL) + pszName = (uchar*) "[localhost]"; + else { + if(GetPreserveFQDN() == 1) + pszName = LocalFQDNName; + else + pszName = LocalHostName; + } + } else { /* local hostname is overriden via config */ + pszName = LocalHostNameOverride; } + DBGPRINTF("GenerateLocalHostName uses '%s'\n", pszName); CHKiRet(prop.SetString(propLocalHostName, pszName, ustrlen(pszName))); CHKiRet(prop.ConstructFinalize(propLocalHostName)); @@ -296,6 +301,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a free(pszDfltNetstrmDrvrCertFile); pszDfltNetstrmDrvrCertFile = NULL; } + if(LocalHostNameOverride != NULL) { + free(LocalHostNameOverride); + LocalHostNameOverride = NULL; + } if(pszWorkDir != NULL) { free(pszWorkDir); pszWorkDir = NULL; @@ -327,6 +336,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"localhostname", 0, eCmdHdlrGetWord, NULL, &LocalHostNameOverride, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL)); @@ -350,6 +360,7 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(pszWorkDir); if(LocalHostName != NULL) free(LocalHostName); + free(LocalHostNameOverride); if(LocalFQDNName != NULL) free(LocalFQDNName); objRelease(prop, CORE_COMPONENT); -- cgit From cc8237736d11b54a3d6089d836da7ccb6972a29c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Dec 2010 12:21:17 +0100 Subject: added $IMUDPSchedulingPolicy and $IMUDPSchedulingPriority config settings --- runtime/glbl.c | 1 + 1 file changed, 1 insertion(+) (limited to 'runtime') diff --git a/runtime/glbl.c b/runtime/glbl.c index b396ed7c..59d1fb0f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -154,6 +154,7 @@ GenerateLocalHostNameProperty(void) uchar *pszName; if(propLocalHostName != NULL) + prop.Destruct(&propLocalHostName); CHKiRet(prop.Construct(&propLocalHostName)); if(LocalHostNameOverride == NULL) { -- cgit From 7742b2182fc017ca4c9fcfe3b26f4c8d68a9bd58 Mon Sep 17 00:00:00 2001 From: Dražen Kačar Date: Mon, 10 Jan 2011 12:39:23 +0100 Subject: improved imudp real-time scheduling support & bugfix The original code had quite some issues, which are fixed by this commit. Also we do more error checking now. Signed-off-by: Rainer Gerhards --- runtime/rsyslog.c | 18 ++++++++++++++++++ runtime/rsyslog.h | 7 +++++++ runtime/stream.c | 8 +++++++- runtime/wtp.c | 6 ++++++ 4 files changed, 38 insertions(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index a9794840..2722fd6c 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -84,6 +84,12 @@ #include "strgen.h" #include "atomic.h" +#ifdef HAVE_PTHREAD_SETSCHEDPARAM +struct sched_param default_sched_param; +pthread_attr_t default_thread_attr; +int default_thr_sched_policy; +#endif + /* forward definitions */ static rsRetVal dfltErrLogger(int, uchar *errMsg); @@ -138,6 +144,18 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) if(iRefCount == 0) { /* init runtime only if not yet done */ +#ifdef HAVE_PTHREAD_SETSCHEDPARAM + CHKiRet(pthread_getschedparam(pthread_self(), + &default_thr_sched_policy, + &default_sched_param)); + CHKiRet(pthread_attr_init(&default_thread_attr)); + CHKiRet(pthread_attr_setschedpolicy(&default_thread_attr, + default_thr_sched_policy)); + CHKiRet(pthread_attr_setschedparam(&default_thread_attr, + &default_sched_param)); + CHKiRet(pthread_attr_setinheritsched(&default_thread_attr, + PTHREAD_EXPLICIT_SCHED)); +#endif if(ppErrObj != NULL) *ppErrObj = "obj"; CHKiRet(objClassInit(NULL)); /* *THIS* *MUST* always be the first class initilizer being called! */ CHKiRet(objGetObjInterface(pObjIF)); /* this provides the root pointer for all other queries */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 3e6ac4af..0742a2ed 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -25,6 +25,7 @@ */ #ifndef INCLUDED_RSYSLOG_H #define INCLUDED_RSYSLOG_H +#include #include "typedefs.h" /* ############################################################# * @@ -411,6 +412,12 @@ typedef enum rsObjectID rsObjID; #define RSFREEOBJ(x) {(x)->OID = OIDrsFreed; free(x);} #endif +#ifdef HAVE_PTHREAD_SETSCHEDPARAM +extern struct sched_param default_sched_param; +extern pthread_attr_t default_thread_attr; +extern int default_thr_sched_policy; +#endif + /* for the time being, we do our own portability handling here. It * looks like autotools either does not yet support checks for it, or diff --git a/runtime/stream.c b/runtime/stream.c index 260b59ef..658aba11 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -669,7 +669,13 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } pThis->pIOBuf = pThis->asyncBuf[0].pBuf; pThis->bStopWriter = 0; - if(pthread_create(&pThis->writerThreadID, NULL, asyncWriterThread, pThis) != 0) + if(pthread_create(&pThis->writerThreadID, +#ifdef HAVE_PTHREAD_SETSCHEDPARAM + &default_thread_attr, +#else + NULL, +#endif + asyncWriterThread, pThis) != 0) DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis); } else { /* we work synchronously, so we need to alloc a fixed pIOBuf */ diff --git a/runtime/wtp.c b/runtime/wtp.c index ece80911..e615fb19 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -90,6 +90,12 @@ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! pthread_mutex_init(&pThis->mutWtp, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); pthread_attr_init(&pThis->attrThrd); + /* Set thread scheduling policy to default */ +#ifdef HAVE_PTHREAD_SETSCHEDPARAM + pthread_attr_setschedpolicy(&pThis->attrThrd, default_thr_sched_policy); + pthread_attr_setschedparam(&pThis->attrThrd, &default_sched_param); + pthread_attr_setinheritsched(&pThis->attrThrd, PTHREAD_EXPLICIT_SCHED); +#endif pthread_attr_setdetachstate(&pThis->attrThrd, PTHREAD_CREATE_DETACHED); /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; -- cgit From 9813a12789d08a00fa96c11f113cd531c7f2ce7b Mon Sep 17 00:00:00 2001 From: David Lang Date: Tue, 25 Jan 2011 14:24:50 +0100 Subject: enhanced imfile to support multi-line messages Signed-off-by: Rainer Gerhards --- runtime/stream.c | 111 ++++++++++++++++++++++++++++++++++++++++++------------- runtime/stream.h | 5 ++- 2 files changed, 88 insertions(+), 28 deletions(-) (limited to 'runtime') diff --git a/runtime/stream.c b/runtime/stream.c index 658aba11..16d41a2e 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -561,39 +561,98 @@ static rsRetVal strmUnreadChar(strm_t *pThis, uchar c) return RS_RET_OK; } - -/* read a line from a strm file. A line is terminated by LF. The LF is read, but it - * is not returned in the buffer (it is discared). The caller is responsible for - * destruction of the returned CStr object! -- rgerhards, 2008-01-07 - * rgerhards, 2008-03-27: I now use the ppCStr directly, without any interim - * string pointer. The reason is that this function my be called by inputs, which - * are pthread_killed() upon termination. So if we use their native pointer, they - * can cleanup (but only then). +/* read a 'paragraph' from a strm file. + * A paragraph may be terminated by a LF, by a LFLF, or by LF depending on the option set. + * The termination LF characters are read, but are + * not returned in the buffer (it is discared). The caller is responsible for + * destruction of the returned CStr object! -- dlang 2010-12-13 */ static rsRetVal -strmReadLine(strm_t *pThis, cstr_t **ppCStr) +strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) { - DEFiRet; - uchar c; - - ASSERT(pThis != NULL); - ASSERT(ppCStr != NULL); - - CHKiRet(cstrConstruct(ppCStr)); - - /* now read the line */ - CHKiRet(strmReadChar(pThis, &c)); - while(c != '\n') { - CHKiRet(cstrAppendChar(*ppCStr, c)); - CHKiRet(strmReadChar(pThis, &c)); + /* mode = 0 single line mode (equivalent to ReadLine) + * mode = 1 LFLF mode (paragraph, blank line between entries) + * mode = 2 LF mode, a log line starts at the beginning of a line, but following lines that are indented are part of the same log entry + * This modal interface is not nearly as flexible as being able to define a regex for when a new record starts, but it's also not nearly as hard (or as slow) to implement + */ + DEFiRet; + uchar c; + uchar finished; + + ASSERT(pThis != NULL); + ASSERT(ppCStr != NULL); + + CHKiRet(cstrConstruct(ppCStr)); + + /* now read the line */ + CHKiRet(strmReadChar(pThis, &c)); + if (mode == 0){ + while(c != '\n') { + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } + CHKiRet(cstrFinalize(*ppCStr)); + } + if (mode == 1){ + finished=0; + while(finished == 0){ + if(c != '\n') { + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { + if ((((*ppCStr)->iStrLen) > 0) ){ + if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] == '\n'){ + rsCStrTruncate(*ppCStr,1); /* remove the prior newline */ + finished=1; + } else { + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } + } else { + finished=1; /* this is a blank line, a \n with nothing since the last complete record */ + } + } + } + CHKiRet(cstrFinalize(*ppCStr)); + } + if (mode == 2){ +/* indented follow-up lines */ + finished=0; + while(finished == 0){ + if ((*ppCStr)->iStrLen == 0){ + if(c != '\n') { +/* nothing in the buffer, and it's not a newline, add it to the buffer */ + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { + finished=1; /* this is a blank line, a \n with nothing since the last complete record */ + } + } else { + if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] != '\n'){ +/* not the first character after a newline, add it to the buffer */ + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { + if ((c == ' ') || (c == '\t')){ + CHKiRet(cstrAppendChar(*ppCStr, c)); + CHKiRet(strmReadChar(pThis, &c)); + } else { +/* clean things up by putting the character we just read back into the input buffer and removing the LF character that is currently at the end of the output string */ + CHKiRet(strmUnreadChar(pThis, c)); + rsCStrTruncate(*ppCStr,1); + finished=1; + } + } + } + } + CHKiRet(cstrFinalize(*ppCStr)); } - CHKiRet(cstrFinalize(*ppCStr)); finalize_it: - if(iRet != RS_RET_OK && *ppCStr != NULL) - cstrDestruct(ppCStr); + if(iRet != RS_RET_OK && *ppCStr != NULL) + cstrDestruct(ppCStr); - RETiRet; + RETiRet; } diff --git a/runtime/stream.h b/runtime/stream.h index 37e9d570..60c68cb2 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -156,7 +156,6 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); rsRetVal (*ReadChar)(strm_t *pThis, uchar *pC); rsRetVal (*UnreadChar)(strm_t *pThis, uchar c); - rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr); rsRetVal (*SeekCurrOffs)(strm_t *pThis); rsRetVal (*Write)(strm_t *pThis, uchar *pBuf, size_t lenBuf); rsRetVal (*WriteChar)(strm_t *pThis, uchar c); @@ -183,8 +182,10 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, iSizeLimit, off_t); INTERFACEpropSetMeth(strm, iFlushInterval, int); INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); + /* v6 added */ + rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr, int mode); ENDinterface(strm) -#define strmCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* prototypes */ -- cgit From df6bad70d4df6b9142f988bd4ed859555756fd15 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Feb 2011 17:46:04 +0100 Subject: cosmetic: making comments in stream.c a bit more readable --- runtime/stream.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'runtime') diff --git a/runtime/stream.c b/runtime/stream.c index 5f4249a8..24dbcc09 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -617,12 +617,12 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) CHKiRet(cstrFinalize(*ppCStr)); } if (mode == 2){ -/* indented follow-up lines */ + /* indented follow-up lines */ finished=0; while(finished == 0){ if ((*ppCStr)->iStrLen == 0){ if(c != '\n') { -/* nothing in the buffer, and it's not a newline, add it to the buffer */ + /* nothing in the buffer, and it's not a newline, add it to the buffer */ CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } else { @@ -630,7 +630,7 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) } } else { if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] != '\n'){ -/* not the first character after a newline, add it to the buffer */ + /* not the first character after a newline, add it to the buffer */ CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } else { @@ -638,7 +638,9 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, int mode) CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } else { -/* clean things up by putting the character we just read back into the input buffer and removing the LF character that is currently at the end of the output string */ + /* clean things up by putting the character we just read back into + * the input buffer and removing the LF character that is currently at the + * end of the output string */ CHKiRet(strmUnreadChar(pThis, c)); rsCStrTruncate(*ppCStr,1); finished=1; -- cgit From 1f66eb6efaf2c4bea18242645c85b51fe5236fb9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 10 Feb 2011 17:50:41 +0100 Subject: bugfix: abort if imfile reads file line of more than 64KiB Thanks to Peter Eisentraut for reporting and analysing this problem. bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=221 --- runtime/stringbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index ccf115c1..2b6815a4 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -156,7 +156,7 @@ rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { uchar *pNewBuf; - unsigned short iNewSize; + size_t iNewSize; DEFiRet; /* first compute the new size needed */ -- cgit From 121f5ab4ec4e766aeae5671005325a6aef4a1806 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 14:56:02 +0100 Subject: bugfix: queue engine did not properly slow down inputs in FULL_DELAY mode... ...when in disk-assisted mode. This especially affected imfile, which created unnecessarily queue files if a large set of input file data was to process. --- runtime/queue.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/queue.c b/runtime/queue.c index e4922f37..cc7c0f54 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -246,6 +246,7 @@ qqueueAdviseMaxWorkers(qqueue_t *pThis) if(!pThis->bEnqOnly) { if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) { + DBGOPRINT((obj_t*) pThis, "(re)activating DA worker\n"); wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } else { if(getLogicalQueueSize(pThis) == 0) { @@ -1211,7 +1212,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread /* set some water marks so that we have useful defaults if none are set specifically */ pThis->iFullDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 3; /* default 97% */ pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */ - pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; @@ -1819,6 +1819,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; uchar pszBuf[64]; + int wrk; uchar *qName; size_t lenBuf; @@ -1850,6 +1851,16 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* call type-specific constructor */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ + /* re-adjust some params if required */ + if(pThis->bIsDA) { + /* if we are in DA mode, we must make sure full delayable messages do not + * initiate going to disk! + */ + wrk = pThis->iHighWtrMrk - (pThis->iHighWtrMrk / 100) * 50; /* 50% of high water mark */ + if(wrk < pThis->iFullDlyMrk) + pThis->iFullDlyMrk = wrk; + } + DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, " "full delay %d, light delay %d, deq batch size %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, -- cgit From 6a18d25cbec2676a7910ff038170716293abe89f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 17:06:20 +0100 Subject: removed no longer needed code --- runtime/queue.c | 2 -- runtime/queue.h | 1 - 2 files changed, 3 deletions(-) (limited to 'runtime') diff --git a/runtime/queue.c b/runtime/queue.c index cc7c0f54..9f63a338 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1842,7 +1842,6 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ } pthread_mutex_init(&pThis->mutThrdMgmt, NULL); - pthread_cond_init (&pThis->condDAReady, NULL); pthread_cond_init (&pThis->notFull, NULL); pthread_cond_init (&pThis->notEmpty, NULL); pthread_cond_init (&pThis->belowFullDlyWtrMrk, NULL); @@ -2140,7 +2139,6 @@ CODESTARTobjDestruct(qqueue) free(pThis->mut); } pthread_mutex_destroy(&pThis->mutThrdMgmt); - pthread_cond_destroy(&pThis->condDAReady); pthread_cond_destroy(&pThis->notFull); pthread_cond_destroy(&pThis->notEmpty); pthread_cond_destroy(&pThis->belowFullDlyWtrMrk); diff --git a/runtime/queue.h b/runtime/queue.h index 38e248cd..97057180 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -124,7 +124,6 @@ struct queue_s { pthread_cond_t notFull, notEmpty; pthread_cond_t belowFullDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ pthread_cond_t belowLightDlyWtrMrk; /* below eFLOWCTL_FULL_DELAY watermark */ - pthread_cond_t condDAReady;/* signalled when the DA queue is fully initialized and ready for processing */ int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* the following variables are always present, because they -- cgit From f3d354da3e373f9c4890a78e5274a6ba02f1c8cb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 11 Feb 2011 17:47:30 +0100 Subject: bugfix: very long running actions could prevent shutdown under some circumstances This has now been solved, at least for common situations. --- runtime/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/queue.c b/runtime/queue.c index 9f63a338..76327f6a 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1497,7 +1497,7 @@ DequeueConsumable(qqueue_t *pThis, wti_t *pWti) * now that we dequeue batches of pointers, this is much less an issue... * rgerhards, 2009-04-22 */ - if(iQueueSize < pThis->iFullDlyMrk / 2) { + if(iQueueSize < pThis->iFullDlyMrk / 2 || glbl.GetGlobalInputTermState() == 1) { pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk); } -- cgit From 49c2bc380c34c346dafffe5c5a4059fa2f604680 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 15 Feb 2011 11:06:15 +0100 Subject: improved error reporting for $WorkDirectory non-existance and other detectable problems are now reported, and the work directory is NOT set in this case --- runtime/glbl.c | 37 ++++++++++++++++++++++++++++++++++++- runtime/rsyslog.h | 1 + 2 files changed, 37 insertions(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/glbl.c b/runtime/glbl.c index c7bb88a5..68eb276c 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -31,6 +31,9 @@ #include "config.h" #include #include +#include +#include +#include #include #include "rsyslog.h" @@ -40,6 +43,7 @@ #include "glbl.h" #include "prop.h" #include "atomic.h" +#include "errmsg.h" /* some defaults */ #ifndef DFLT_NETSTRM_DRVR @@ -49,6 +53,7 @@ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(prop) +DEFobjCurrIf(errmsg) /* static data * For this object, these variables are obviously what makes the "meat" of the @@ -147,6 +152,35 @@ static void SetGlobalInputTermination(void) } +/* This function is used to set the global work directory name. + * It verifies that the provided directory actually exists and + * emits an error message if not. + * rgerhards, 2011-02-16 + */ +static rsRetVal setWorkDir(void __attribute__((unused)) *pVal, uchar *pNewVal) +{ + DEFiRet; + struct stat sb; + + if(stat((char*) pNewVal, &sb) != 0) { + errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: %s can not be " + "accessed, probably does not exist - directive ignored", pNewVal); + ABORT_FINALIZE(RS_RET_ERR_WRKDIR); + } + + if(!S_ISDIR(sb.st_mode)) { + errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: %s not a directory - directive ignored", + pNewVal); + ABORT_FINALIZE(RS_RET_ERR_WRKDIR); + } + + free(pszWorkDir); + pszWorkDir = pNewVal; + +finalize_it: + RETiRet; +} + /* return our local hostname. if it is not set, "[localhost]" is returned */ static uchar* @@ -354,9 +388,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* register config handlers (TODO: we need to implement a way to unregister them) */ - CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, NULL, &pszWorkDir, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, setWorkDir, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"dropmsgswithmaliciousdnsptrrecords", 0, eCmdHdlrBinary, NULL, &bDropMalPTRMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 0402f159..78e61bcb 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -340,6 +340,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_HDFS_WRITE = -2178, /**< error writing to HDFS */ RS_RET_ERR_HDFS_OPEN = -2179, /**< error during hdfsOpen (e.g. file does not exist) */ RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */ + RS_RET_ERR_WRKDIR = -2181, /**< problems with the rsyslog working directory */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit From 7052d3c378ff600f9f03a05b1918d8eda6b76e2b Mon Sep 17 00:00:00 2001 From: Dražen Kačar Date: Wed, 16 Feb 2011 18:20:59 +0100 Subject: bugfix: fixed compile problem due to empty structs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this occured only on some platforms/compilers. thanks to Dražen Kačar for the fix Signed-off-by: Rainer Gerhards --- runtime/datetime.h | 1 + runtime/errmsg.h | 1 + runtime/expr.h | 1 + runtime/modules.h | 1 + 4 files changed, 4 insertions(+) (limited to 'runtime') diff --git a/runtime/datetime.h b/runtime/datetime.h index 82bd415b..70bbf416 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -28,6 +28,7 @@ /* the datetime object */ typedef struct datetime_s { + char dummy; } datetime_t; diff --git a/runtime/errmsg.h b/runtime/errmsg.h index 799954fb..ac7018b3 100644 --- a/runtime/errmsg.h +++ b/runtime/errmsg.h @@ -30,6 +30,7 @@ /* the errmsg object */ typedef struct errmsg_s { + char dummy; } errmsg_t; diff --git a/runtime/expr.h b/runtime/expr.h index 974b71ec..1afe1a1f 100644 --- a/runtime/expr.h +++ b/runtime/expr.h @@ -30,6 +30,7 @@ /* a node inside an expression tree */ typedef struct exprNode_s { + char dummy; } exprNode_t; diff --git a/runtime/modules.h b/runtime/modules.h index 49586e8d..df1afbc3 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -119,6 +119,7 @@ struct modInfo_s { rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); } om; struct { /* data for library modules */ + char dummy; } lm; struct { /* data for parser modules */ rsRetVal (*parse)(msg_t*); -- cgit From e867cb41372e9b8e08b8eec0d663a1f3ccea5367 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 23 Feb 2011 08:54:31 +0100 Subject: added debug support for trying to find well-hidden bug --- runtime/msg.c | 10 ++++++++++ runtime/msg.h | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/msg.c b/runtime/msg.c index e8be79db..fb4d5742 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -677,6 +677,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) /* initialize members in ORDER they appear in structure (think "cache line"!) */ pM->flowCtlType = 0; pM->bDoLock = 0; + pM->bAlreadyFreed = 0; pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; @@ -803,6 +804,15 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ + /* The if below is included to try to nail down a well-hidden bug causing + * segfaults. I hope that do to the test code the problem is sooner detected and + * thus we get better data for debugging and resolving it. -- rgerhards, 2011-02-23. + * TODO: remove when no longer needed. + */ + if(pThis->bAlreadyFreed) + abort(); + pThis->bAlreadyFreed = 1; + /* end debug code */ if(pThis->pszRawMsg != pThis->szRawMsg) free(pThis->pszRawMsg); freeTAG(pThis); diff --git a/runtime/msg.h b/runtime/msg.h index 4897959c..26a07aca 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -61,7 +61,8 @@ struct msg { once data has entered the queue, this property is no longer needed. */ pthread_mutex_t mut; int iRefCount; /* reference counter (0 = unused) */ - sbool bDoLock; /* use the mutex? */ + sbool bDoLock; /* use the mutex? */ + sbool bAlreadyFreed; /* aid to help detect a well-hidden bad bug -- TODO: remove when no longer needed */ short iSeverity; /* the severity 0..7 */ short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ -- cgit From d1eb6e0edc51a78f3209448e800b25eda50340f2 Mon Sep 17 00:00:00 2001 From: Bojan Smojver Date: Wed, 23 Feb 2011 11:25:43 +0100 Subject: added work-around for bug in gtls, which causes fd leak when using TLS The capability has been added for module to specify that they do not like being unloaded. related bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=222 Signed-off-by: Rainer Gerhards --- runtime/module-template.h | 12 ++++++++++++ runtime/modules.c | 48 +++++++++++++++++++++++++++++++++++++++++++---- runtime/modules.h | 14 ++++++++++++++ runtime/net.c | 1 + runtime/netstrms.c | 1 + runtime/nsd_gtls.c | 1 + runtime/nsd_ptcp.c | 1 + runtime/regexp.c | 1 + runtime/strmsrv.c | 1 + runtime/zlibw.c | 1 + 10 files changed, 77 insertions(+), 4 deletions(-) (limited to 'runtime') diff --git a/runtime/module-template.h b/runtime/module-template.h index d05ec23c..c2585e67 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -77,6 +77,16 @@ static rsRetVal modGetType(eModType_t *modType) \ DEF_LMOD_STATIC_DATA \ MODULE_TYPE(eMOD_LIB) +/* Macro to define whether the module should be kept dynamically linked. + */ +#define MODULE_KEEP_TYPE(x)\ +static rsRetVal modGetKeepType(eModKeepType_t *modKeepType) \ + { \ + *modKeepType = x; \ + return RS_RET_OK;\ + } +#define MODULE_TYPE_NOKEEP MODULE_KEEP_TYPE(eMOD_NOKEEP) +#define MODULE_TYPE_KEEP MODULE_KEEP_TYPE(eMOD_KEEP) /* macro to define a unique module id. This must be able to fit in a void*. The * module id must be unique inside a running rsyslogd application. It is used to @@ -342,6 +352,8 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = modGetID;\ } else if(!strcmp((char*) name, "getType")) {\ *pEtryPoint = modGetType;\ + } else if(!strcmp((char*) name, "getKeepType")) {\ + *pEtryPoint = modGetKeepType;\ } /* the following definition is the standard block for queryEtryPt for output diff --git a/runtime/modules.c b/runtime/modules.c index d7362753..86e7c695 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -77,6 +77,9 @@ static pthread_mutex_t mutLoadUnload; static modInfo_t *pLoadedModules = NULL; /* list of currently-loaded modules */ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ +/* already dlopen()-ed libs */ +static struct dlhandle_s *pHandles = NULL; + /* config settings */ uchar *pModDir = NULL; /* read-only after startup */ @@ -232,7 +235,9 @@ static void moduleDestruct(modInfo_t *pThis) # ifdef VALGRIND # warning "dlclose disabled for valgrind" # else - dlclose(pThis->pModHdlr); + if (pThis->eKeepType == eMOD_NOKEEP) { + dlclose(pThis->pModHdlr); + } # endif } @@ -413,6 +418,8 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ strgen_t *pStrgen; /* used for strgen modules */ rsRetVal (*GetName)(uchar**); rsRetVal (*modGetType)(eModType_t *pType); + rsRetVal (*modGetKeepType)(eModKeepType_t *pKeepType); + struct dlhandle_s *pHandle = NULL; DEFiRet; assert(modInit != NULL); @@ -433,6 +440,8 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ */ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getType", &modGetType)); CHKiRet((*modGetType)(&pNew->eType)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"getKeepType", &modGetKeepType)); + CHKiRet((*modGetKeepType)(&pNew->eKeepType)); dbgprintf("module of type %d being loaded.\n", pNew->eType); /* OK, we know we can successfully work with the module. So we now fill the @@ -529,11 +538,28 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ pNew->pszName = (uchar*) strdup((char*)name); /* we do not care if strdup() fails, we can accept that */ pNew->pModHdlr = pModHdlr; /* TODO: take this from module */ - if(pModHdlr == NULL) + if(pModHdlr == NULL) { pNew->eLinkType = eMOD_LINK_STATIC; - else + } else { pNew->eLinkType = eMOD_LINK_DYNAMIC_LOADED; + /* if we need to keep the linked module, save it */ + if (pNew->eKeepType == eMOD_KEEP) { + if((pHandle = calloc(1, sizeof (*pHandle))) == NULL) { + iRet = RS_RET_OUT_OF_MEMORY; + goto finalize_it; + } + + strncpy((char *)pHandle->szName, + (char *)name, PATH_MAX - 1); + pHandle->szName[PATH_MAX - 1] = '\0'; + pHandle->pModHdlr = pModHdlr; + pHandle->next = pHandles; + + pHandles = pHandle; + } + } + /* we initialized the structure, now let's add it to the linked list of modules */ addModToList(pNew); @@ -740,6 +766,7 @@ Load(uchar *pModName) modInfo_t *pModInfo; uchar *pModDirCurr, *pModDirNext; int iLoadCnt; + struct dlhandle_s *pHandle = NULL; assert(pModName != NULL); dbgprintf("Requested to load module '%s'\n", pModName); @@ -829,7 +856,20 @@ Load(uchar *pModName) /* complete load path constructed, so ... GO! */ dbgprintf("loading module '%s'\n", szPath); - pModHdlr = dlopen((char *) szPath, RTLD_NOW); + + /* see if we have this one already */ + for (pHandle = pHandles; pHandle; pHandle = pHandle->next) { + if (!strcmp((char *)pModName, (char *)pHandle->szName)) { + pModHdlr = pHandle->pModHdlr; + break; + } + } + + /* not found, try to dynamically link it */ + if (!pModHdlr) { + pModHdlr = dlopen((char *) szPath, RTLD_NOW); + } + iLoadCnt++; } while(pModHdlr == NULL && *pModName != '/' && pModDirNext); diff --git a/runtime/modules.h b/runtime/modules.h index df1afbc3..7eff8581 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -75,12 +75,26 @@ typedef enum eModLinkType_ { eMOD_LINK_ALL /* special: all linkage types, e.g. for unload */ } eModLinkType_t; +/* remember which shared libs we dlopen()-ed */ +struct dlhandle_s { + uchar szName[PATH_MAX]; + void *pModHdlr; + struct dlhandle_s *next; +}; + +/* should this module be kept linked? */ +typedef enum eModKeepType_ { + eMOD_NOKEEP, + eMOD_KEEP +} eModKeepType_t; + struct modInfo_s { struct modInfo_s *pPrev; /* support for creating a double linked module list */ struct modInfo_s *pNext; /* support for creating a linked module list */ int iIFVers; /* Interface version of module */ eModType_t eType; /* type of this module */ eModLinkType_t eLinkType; + eModKeepType_t eKeepType; /* keep the module dynamically linked on unload */ uchar* pszName; /* printable module name, e.g. for dbgprintf */ unsigned uRefCnt; /* reference count for this module; 0 -> may be unloaded */ /* functions supported by all types of modules */ diff --git a/runtime/net.c b/runtime/net.c index 7653ea1d..789790f6 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -69,6 +69,7 @@ #endif MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/netstrms.c b/runtime/netstrms.c index e9ff2568..ea2dd9f3 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -40,6 +40,7 @@ #include "netstrms.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 0ee70e56..152dc8de 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -56,6 +56,7 @@ GCRY_THREAD_OPTION_PTHREAD_IMPL; MODULE_TYPE_LIB +MODULE_TYPE_KEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index ca00749c..c8915231 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -52,6 +52,7 @@ #include "nsd_ptcp.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/regexp.c b/runtime/regexp.c index 86b3e6c4..21079f80 100644 --- a/runtime/regexp.c +++ b/runtime/regexp.c @@ -35,6 +35,7 @@ #include "regexp.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index a122ca8a..e66ad717 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -75,6 +75,7 @@ #include "unicode-helper.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* defines */ #define STRMSESS_MAX_DEFAULT 200 /* default for nbr of strm sessions if no number is given */ diff --git a/runtime/zlibw.c b/runtime/zlibw.c index 2b386213..455c20d9 100644 --- a/runtime/zlibw.c +++ b/runtime/zlibw.c @@ -34,6 +34,7 @@ #include "zlibw.h" MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP /* static data */ DEFobjStaticHelpers -- cgit From 1ef709cc97d54f74d3fdeb83788cc4b01f4c6a2a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 25 Feb 2011 14:14:17 +0100 Subject: bugfix: fixed a memory leak and potential abort condition this could happen if multiple rulesets were used and some output batches contained messages belonging to more than one ruleset. fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=226 fixes: http://bugzilla.adiscon.com/show_bug.cgi?id=218 --- runtime/batch.h | 8 +++++++- runtime/ruleset.c | 23 ++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) (limited to 'runtime') diff --git a/runtime/batch.h b/runtime/batch.h index d0504f2b..944889bd 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -136,11 +136,16 @@ batchIsValidElem(batch_t *pBatch, int i) { /* copy one batch element to another. * This creates a complete duplicate in those cases where * it is needed. Use duplication only when absolutely necessary! + * Note that all working fields are reset to zeros. If that were + * not done, we would have potential problems with invalid + * or double pointer frees. * rgerhards, 2010-06-10 */ static inline void batchCopyElem(batch_obj_t *pDest, batch_obj_t *pSrc) { - memcpy(pDest, pSrc, sizeof(batch_obj_t)); + memset(pDest, 0, sizeof(batch_obj_t)); + pDest->pUsrp = pSrc->pUsrp; + pDest->state = pSrc->state; } @@ -171,6 +176,7 @@ batchFree(batch_t *pBatch) { static inline rsRetVal batchInit(batch_t *pBatch, int maxElem) { DEFiRet; + pBatch->iDoneUpTo = 0; pBatch->maxElem = maxElem; CHKmalloc(pBatch->pElem = calloc((size_t)maxElem, sizeof(batch_obj_t))); // TODO: replace calloc by inidividual writes? diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 0584e8d6..c9c64a38 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -171,35 +171,40 @@ processBatchMultiRuleset(batch_t *pBatch) int i; int iStart; /* start index of partial batch */ int iNew; /* index for new (temporary) batch */ + int bHaveUnprocessed; /* do we (still) have unprocessed entries? (loop term predicate) */ DEFiRet; - CHKiRet(batchInit(&snglRuleBatch, pBatch->nElem)); - snglRuleBatch.pbShutdownImmediate = pBatch->pbShutdownImmediate; - - while(1) { /* loop broken inside */ + do { + bHaveUnprocessed = 0; /* search for first unprocessed element */ for(iStart = 0 ; iStart < pBatch->nElem && pBatch->pElem[iStart].state == BATCH_STATE_DISC ; ++iStart) /* just search, no action */; - if(iStart == pBatch->nElem) - FINALIZE; /* everything processed */ + break; /* everything processed */ /* prepare temporary batch */ + CHKiRet(batchInit(&snglRuleBatch, pBatch->nElem)); + snglRuleBatch.pbShutdownImmediate = pBatch->pbShutdownImmediate; currRuleset = batchElemGetRuleset(pBatch, iStart); iNew = 0; for(i = iStart ; i < pBatch->nElem ; ++i) { if(batchElemGetRuleset(pBatch, i) == currRuleset) { - batchCopyElem(&(snglRuleBatch.pElem[iNew++]), &(pBatch->pElem[i])); + /* for performance reasons, we copy only those members that we actually need */ + snglRuleBatch.pElem[iNew].pUsrp = pBatch->pElem[i].pUsrp; + snglRuleBatch.pElem[iNew].state = pBatch->pElem[i].state; + ++iNew; /* We indicate the element also as done, so it will not be processed again */ pBatch->pElem[i].state = BATCH_STATE_DISC; + } else { + bHaveUnprocessed = 1; } } snglRuleBatch.nElem = iNew; /* was left just right by the for loop */ batchSetSingleRuleset(&snglRuleBatch, 1); /* process temp batch */ processBatch(&snglRuleBatch); - } - batchFree(&snglRuleBatch); + batchFree(&snglRuleBatch); + } while(bHaveUnprocessed == 1); finalize_it: RETiRet; -- cgit From bd73f263b310c29d5e0b0dd541403bde44030b86 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 1 Mar 2011 14:23:21 +0100 Subject: bugfix: potential abort condition when $RepeatedMsgReduction set to on as well as potentially in a number of other places where MsgDup() was used. This only happened when the imudp input module was used and it depended on name resolution not yet had taken place. In other words, this was a strange problem that could lead to hard to diagnose instability. So if you experience instability, chances are good that this fix will help. --- runtime/msg.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'runtime') diff --git a/runtime/msg.c b/runtime/msg.c index fb4d5742..b0261faa 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -932,13 +932,14 @@ msg_t* MsgDup(msg_t* pOld) pNew->iLenMSG = pOld->iLenMSG; pNew->iLenTAG = pOld->iLenTAG; pNew->iLenHOSTNAME = pOld->iLenHOSTNAME; - if((pOld->msgFlags & NEEDS_DNSRESOL) == 1) { + if((pOld->msgFlags & NEEDS_DNSRESOL)) { localRet = msgSetFromSockinfo(pNew, pOld->rcvFrom.pfrominet); if(localRet != RS_RET_OK) { /* if something fails, we accept loss of this property, it is * better than losing the whole message. */ pNew->msgFlags &= ~NEEDS_DNSRESOL; + pNew->rcvFrom.pRcvFrom = NULL; /* make sure no dangling values */ } } else { if(pOld->rcvFrom.pRcvFrom != NULL) { -- cgit From 87f8ac6dc4343eb4664ca5e234c163c399afca1b Mon Sep 17 00:00:00 2001 From: Bojan Smojver Date: Fri, 4 Mar 2011 07:55:53 +0100 Subject: bugfix: regression: memory leak in module loader Signed-off-by: Rainer Gerhards --- runtime/modules.c | 28 ++++++++++++++++++---------- runtime/modules.h | 6 +++--- 2 files changed, 21 insertions(+), 13 deletions(-) (limited to 'runtime') diff --git a/runtime/modules.c b/runtime/modules.c index 86e7c695..4541bddf 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -545,18 +545,26 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ /* if we need to keep the linked module, save it */ if (pNew->eKeepType == eMOD_KEEP) { - if((pHandle = calloc(1, sizeof (*pHandle))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto finalize_it; + /* see if we have this one already */ + for (pHandle = pHandles; pHandle; pHandle = pHandle->next) { + if (!strcmp((char *)name, (char *)pHandle->pszName)) + break; } - strncpy((char *)pHandle->szName, - (char *)name, PATH_MAX - 1); - pHandle->szName[PATH_MAX - 1] = '\0'; - pHandle->pModHdlr = pModHdlr; - pHandle->next = pHandles; + /* not found, create it */ + if (!pHandle) { + if((pHandle = malloc(sizeof (*pHandle))) == NULL) { + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + if((pHandle->pszName = (uchar*) strdup((char*)name)) == NULL) { + free(pHandle); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + pHandle->pModHdlr = pModHdlr; + pHandle->next = pHandles; - pHandles = pHandle; + pHandles = pHandle; + } } } @@ -859,7 +867,7 @@ Load(uchar *pModName) /* see if we have this one already */ for (pHandle = pHandles; pHandle; pHandle = pHandle->next) { - if (!strcmp((char *)pModName, (char *)pHandle->szName)) { + if (!strcmp((char *)pModName, (char *)pHandle->pszName)) { pModHdlr = pHandle->pModHdlr; break; } diff --git a/runtime/modules.h b/runtime/modules.h index 7eff8581..4daaf1f9 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -77,9 +77,9 @@ typedef enum eModLinkType_ { /* remember which shared libs we dlopen()-ed */ struct dlhandle_s { - uchar szName[PATH_MAX]; - void *pModHdlr; - struct dlhandle_s *next; + uchar *pszName; + void *pModHdlr; + struct dlhandle_s *next; }; /* should this module be kept linked? */ -- cgit From 4671cad51221f1cb03822481e46669ffba3c6d95 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Sun, 13 Mar 2011 12:09:06 +0100 Subject: Fall back to epoll_create() if epoll_create1() is not available epoll_create1() was introduced in Linux kernel 2.6.27. If rsyslog was compiled on a newer kernel but run on a kernel older than 2.6.27, remote syslog fails. Apply a runtime check for epoll_create1() and fall back to epoll_create() in this case. Bug-Debian: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=617996 --- runtime/nsdpoll_ptcp.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'runtime') diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c index bc374c60..ef9c37a3 100644 --- a/runtime/nsdpoll_ptcp.c +++ b/runtime/nsdpoll_ptcp.c @@ -133,13 +133,16 @@ delEvent(nsdpoll_epollevt_lst_t **ppEvtLst) { /* Standard-Constructor */ BEGINobjConstruct(nsdpoll_ptcp) /* be sure to specify the object type also in END macro! */ -# if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) - DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); - pThis->efd = epoll_create1(EPOLL_CLOEXEC); -# else +#if defined(EPOLL_CLOEXEC) && defined(HAVE_EPOLL_CREATE1) + DBGPRINTF("nsdpoll_ptcp uses epoll_create1()\n"); + pThis->efd = epoll_create1(EPOLL_CLOEXEC); + if(pThis->efd < 0 && errno == ENOSYS) +#endif + { DBGPRINTF("nsdpoll_ptcp uses epoll_create()\n"); pThis->efd = epoll_create(100); /* size is ignored in newer kernels, but 100 is not bad... */ -# endif + } + if(pThis->efd < 0) { DBGPRINTF("epoll_create1() could not create fd\n"); ABORT_FINALIZE(RS_RET_IO_ERROR); -- cgit From 32db6291bc310062cffd3266fc297c7b13c571a7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 16 Mar 2011 16:59:49 +0100 Subject: backporting patch for clean make distcheck --- runtime/cfsysline.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'runtime') diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index 037e9f84..646ab39b 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -41,6 +41,7 @@ #include "obj.h" #include "errmsg.h" #include "srUtils.h" +#include "unicode-helper.h" /* static data */ @@ -511,6 +512,8 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); pStrB = NULL; + DBGPRINTF("doGetWord: get newval '%s' (len %d), hdlr %p\n", + pNewVal, (int) ustrlen(pNewVal), pSetHdlr); /* we got the word, now set it */ if(pSetHdlr == NULL) { /* we should set value directly to var */ -- cgit From e69efbb6d7907339ef81a2062f858eb082233f61 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 22 Mar 2011 16:55:11 +0100 Subject: enhance: added $BOM system property to ease writing byte order masks --- runtime/msg.c | 10 ++++++++++ runtime/rsyslog.h | 1 + 2 files changed, 11 insertions(+) (limited to 'runtime') diff --git a/runtime/msg.c b/runtime/msg.c index b0261faa..e8e60963 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -444,6 +444,8 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) *pPropID = PROP_SYS_MINUTE; } else if(!strcmp((char*) pName, "$myhostname")) { *pPropID = PROP_SYS_MYHOSTNAME; + } else if(!strcmp((char*) pName, "$bom")) { + *pPropID = PROP_SYS_BOM; } else { *pPropID = PROP_INVALID; iRet = RS_RET_VAR_NOT_FOUND; @@ -525,6 +527,8 @@ uchar *propIDToName(propid_t propID) return UCHAR_CONSTANT("$MINUTE"); case PROP_SYS_MYHOSTNAME: return UCHAR_CONSTANT("$MYHOSTNAME"); + case PROP_SYS_BOM: + return UCHAR_CONSTANT("$BOM"); default: return UCHAR_CONSTANT("*invalid property id*"); } @@ -2427,6 +2431,12 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, case PROP_SYS_MYHOSTNAME: pRes = glbl.GetLocalHostName(); break; + case PROP_SYS_BOM: + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (uchar*) "\xEF\xBB\xBF"; + *pbMustBeFreed = 0; + break; default: /* there is no point in continuing, we may even otherwise render the * error message unreadable. rgerhards, 2007-07-10 diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 78e61bcb..d63dbe4f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -137,6 +137,7 @@ typedef uintTiny propid_t; #define PROP_SYS_QHOUR 156 #define PROP_SYS_MINUTE 157 #define PROP_SYS_MYHOSTNAME 158 +#define PROP_SYS_BOM 159 /* The error codes below are orginally "borrowed" from -- cgit From eefc9b6eb69c130015955cfb15dc45e36e431d5a Mon Sep 17 00:00:00 2001 From: Corey Smith Date: Thu, 24 Mar 2011 17:25:03 +0100 Subject: bugfix: PRI was invalid on Solaris for message from local log socket Signed-off-by: root --- runtime/parser.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'runtime') diff --git a/runtime/parser.c b/runtime/parser.c index d3644976..b385c54b 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -453,10 +453,10 @@ ParsePRI(msg_t *pMsg) if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) pri = DEFUPRI; } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); } - pMsg->iFacility = LOG_FAC(pri); - pMsg->iSeverity = LOG_PRI(pri); - MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); RETiRet; } -- cgit From 00e1f24187ee814801e6969629b82a7ae030beaf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 30 Mar 2011 12:13:14 +0200 Subject: stop adding data to DA queue when low watermark has been reached potentially closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241 But needs more verification. --- runtime/queue.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'runtime') diff --git a/runtime/queue.c b/runtime/queue.c index ef6e843b..50ae307c 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1774,9 +1774,13 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) { DEFiRet; +//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis)); if(pThis->bEnqOnly) { iRet = RS_RET_TERMINATE_WHEN_IDLE; } + if(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk) { + iRet = RS_RET_TERMINATE_NOW; + } RETiRet; } -- cgit From 770070a3caf22aee66b18bb98320a59d0ad41fa1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 8 Apr 2011 11:04:38 +0200 Subject: bugfix: race condition in deferred name resolution Note that this actually is a very small change, but I needed to shuffle a lot of code around in order to make it compile (due to required define order...). --- runtime/msg.c | 232 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 117 insertions(+), 115 deletions(-) (limited to 'runtime') diff --git a/runtime/msg.c b/runtime/msg.c index e8e60963..d1e67aa2 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -287,6 +287,121 @@ static pthread_mutex_t mutTrimCtr; /* mutex to handle malloc trim */ static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); +/* The following functions will support advanced output module + * multithreading, once this is implemented. Currently, we + * include them as hooks only. The idea is that we need to guard + * some msg objects data fields against concurrent access if + * we run on multiple threads. Please note that in any case this + * is not necessary for calls from INPUT modules, because they + * construct the message object and do this serially. Only when + * the message is in the processing queue, multiple threads may + * access a single object. Consequently, there are no guard functions + * for "set" methods, as these are called during input. Only "get" + * functions that modify important structures have them. + * rgerhards, 2007-07-20 + * We now support locked and non-locked operations, depending on + * the configuration of rsyslog. To support this, we use function + * pointers. Initially, we start in non-locked mode. There, all + * locking operations call into dummy functions. When locking is + * enabled, the function pointers are changed to functions doing + * actual work. We also introduced another MsgPrepareEnqueue() function + * which initializes the locking structures, if needed. This is + * necessary because internal messages during config file startup + * processing are always created in non-locking mode. So we can + * not initialize locking structures during constructions. We now + * postpone this until when the message is fully constructed and + * enqueued. Then we know the status of locking. This has a nice + * side effect, and that is that during the initial creation of + * the Msg object no locking needs to be done, which results in better + * performance. -- rgerhards, 2008-01-05 + */ +static void (*funcLock)(msg_t *pMsg); +static void (*funcUnlock)(msg_t *pMsg); +static void (*funcDeleteMutex)(msg_t *pMsg); +void (*funcMsgPrepareEnqueue)(msg_t *pMsg); +#if 1 /* This is a debug aid */ +#define MsgLock(pMsg) funcLock(pMsg) +#define MsgUnlock(pMsg) funcUnlock(pMsg) +#else +#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; } +#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); } +#endif + +/* the next function is a dummy to be used by the looking functions + * when the class is not yet running in an environment where locking + * is necessary. Please note that the need to lock can (and will) change + * during a single run. Typically, this is depending on the operation mode + * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05 + */ +static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) +{ + /* empty be design */ +} + + +/* The following function prepares a message for enqueue into the queue. This is + * where a message may be accessed by multiple threads. This implementation here + * is the version for multiple concurrent acces. It initializes the locking + * structures. + * TODO: change to an iRet interface! -- rgerhards, 2008-07-14 + */ +static void MsgPrepareEnqueueLockingCase(msg_t *pThis) +{ + BEGINfunc + assert(pThis != NULL); + pthread_mutex_init(&pThis->mut, NULL); + pThis->bDoLock = 1; + ENDfunc +} + + +/* ... and now the locking and unlocking implementations: */ +static void MsgLockLockingCase(msg_t *pThis) +{ + /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ + assert(pThis != NULL); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_lock(&pThis->mut); +} + +static void MsgUnlockLockingCase(msg_t *pThis) +{ + /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ + assert(pThis != NULL); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_unlock(&pThis->mut); +} + +/* delete the mutex object on message destruction (locking case) + */ +static void MsgDeleteMutexLockingCase(msg_t *pThis) +{ + assert(pThis != NULL); + pthread_mutex_destroy(&pThis->mut); +} + +/* enable multiple concurrent access on the message object + * This works on a class-wide basis and can bot be undone. + * That is, if it is once enabled, it can not be disabled during + * the same run. When this function is called, no other thread + * must manipulate message objects. Then we would have race conditions, + * but guarding against this is counter-productive because it + * would cost additional time. Plus, it would be a programming error. + * rgerhards, 2008-01-05 + */ +rsRetVal MsgEnableThreadSafety(void) +{ + DEFiRet; + funcLock = MsgLockLockingCase; + funcUnlock = MsgUnlockLockingCase; + funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase; + funcDeleteMutex = MsgDeleteMutexLockingCase; + RETiRet; +} + +/* end locking functions */ + + static inline int getProtocolVersion(msg_t *pM) { return(pM->iProtocolVersion); @@ -306,6 +421,7 @@ resolveDNS(msg_t *pMsg) { uchar fromHostFQDN[NI_MAXHOST]; DEFiRet; + MsgLock(pMsg); CHKiRet(objUse(net, CORE_COMPONENT)); if(pMsg->msgFlags & NEEDS_DNSRESOL) { localRet = net.cvthname(pMsg->rcvFrom.pfrominet, fromHost, fromHostFQDN, fromHostIP); @@ -315,6 +431,7 @@ resolveDNS(msg_t *pMsg) { } } finalize_it: + MsgUnlock(pMsg); if(iRet != RS_RET_OK) { /* best we can do: remove property */ MsgSetRcvFromStr(pMsg, UCHAR_CONSTANT(""), 0, &propFromHost); @@ -535,121 +652,6 @@ uchar *propIDToName(propid_t propID) } -/* The following functions will support advanced output module - * multithreading, once this is implemented. Currently, we - * include them as hooks only. The idea is that we need to guard - * some msg objects data fields against concurrent access if - * we run on multiple threads. Please note that in any case this - * is not necessary for calls from INPUT modules, because they - * construct the message object and do this serially. Only when - * the message is in the processing queue, multiple threads may - * access a single object. Consequently, there are no guard functions - * for "set" methods, as these are called during input. Only "get" - * functions that modify important structures have them. - * rgerhards, 2007-07-20 - * We now support locked and non-locked operations, depending on - * the configuration of rsyslog. To support this, we use function - * pointers. Initially, we start in non-locked mode. There, all - * locking operations call into dummy functions. When locking is - * enabled, the function pointers are changed to functions doing - * actual work. We also introduced another MsgPrepareEnqueue() function - * which initializes the locking structures, if needed. This is - * necessary because internal messages during config file startup - * processing are always created in non-locking mode. So we can - * not initialize locking structures during constructions. We now - * postpone this until when the message is fully constructed and - * enqueued. Then we know the status of locking. This has a nice - * side effect, and that is that during the initial creation of - * the Msg object no locking needs to be done, which results in better - * performance. -- rgerhards, 2008-01-05 - */ -static void (*funcLock)(msg_t *pMsg); -static void (*funcUnlock)(msg_t *pMsg); -static void (*funcDeleteMutex)(msg_t *pMsg); -void (*funcMsgPrepareEnqueue)(msg_t *pMsg); -#if 1 /* This is a debug aid */ -#define MsgLock(pMsg) funcLock(pMsg) -#define MsgUnlock(pMsg) funcUnlock(pMsg) -#else -#define MsgLock(pMsg) {dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; } -#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); } -#endif - -/* the next function is a dummy to be used by the looking functions - * when the class is not yet running in an environment where locking - * is necessary. Please note that the need to lock can (and will) change - * during a single run. Typically, this is depending on the operation mode - * of the message queues (which is operator-configurable). -- rgerhards, 2008-01-05 - */ -static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) -{ - /* empty be design */ -} - - -/* The following function prepares a message for enqueue into the queue. This is - * where a message may be accessed by multiple threads. This implementation here - * is the version for multiple concurrent acces. It initializes the locking - * structures. - * TODO: change to an iRet interface! -- rgerhards, 2008-07-14 - */ -static void MsgPrepareEnqueueLockingCase(msg_t *pThis) -{ - BEGINfunc - assert(pThis != NULL); - pthread_mutex_init(&pThis->mut, NULL); - pThis->bDoLock = 1; - ENDfunc -} - - -/* ... and now the locking and unlocking implementations: */ -static void MsgLockLockingCase(msg_t *pThis) -{ - /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ - assert(pThis != NULL); - if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ - pthread_mutex_lock(&pThis->mut); -} - -static void MsgUnlockLockingCase(msg_t *pThis) -{ - /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ - assert(pThis != NULL); - if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ - pthread_mutex_unlock(&pThis->mut); -} - -/* delete the mutex object on message destruction (locking case) - */ -static void MsgDeleteMutexLockingCase(msg_t *pThis) -{ - assert(pThis != NULL); - pthread_mutex_destroy(&pThis->mut); -} - -/* enable multiple concurrent access on the message object - * This works on a class-wide basis and can bot be undone. - * That is, if it is once enabled, it can not be disabled during - * the same run. When this function is called, no other thread - * must manipulate message objects. Then we would have race conditions, - * but guarding against this is counter-productive because it - * would cost additional time. Plus, it would be a programming error. - * rgerhards, 2008-01-05 - */ -rsRetVal MsgEnableThreadSafety(void) -{ - DEFiRet; - funcLock = MsgLockLockingCase; - funcUnlock = MsgUnlockLockingCase; - funcMsgPrepareEnqueue = MsgPrepareEnqueueLockingCase; - funcDeleteMutex = MsgDeleteMutexLockingCase; - RETiRet; -} - -/* end locking functions */ - - /* This is common code for all Constructors. It is defined in an * inline'able function so that we can save a function call in the * actual constructors (otherwise, the msgConstruct would need -- cgit From 9348c80744b29fb5f91b5d8edd3f9070f0d0347b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 Apr 2011 10:35:00 +0200 Subject: cleanup: initially thought temporary debug message made durable ;) It has proven to be very valuable, so now it is properly formatted inside the source. --- runtime/ruleset.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime') diff --git a/runtime/ruleset.c b/runtime/ruleset.c index c9c64a38..5ee2a55a 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -223,7 +223,7 @@ processBatch(batch_t *pBatch) DEFiRet; assert(pBatch != NULL); -dbgprintf("ZZZ: processBatch: batch of %d elements must be processed\n", pBatch->nElem); + DBGPRINTF("processBatch: batch of %d elements must be processed\n", pBatch->nElem); if(pBatch->bSingleRuleset) { pThis = batchGetRuleset(pBatch); if(pThis == NULL) @@ -235,7 +235,7 @@ dbgprintf("ZZZ: processBatch: batch of %d elements must be processed\n", pBatch- } finalize_it: -dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet); + DBGPRINTF("ruleset.ProcessMsg() returns %d\n", iRet); RETiRet; } -- cgit