Initial import of logactio

This is the first step of the logactio framework

Signed-off-by: David Sommerseth <davids@redhat.com>

8 files changed, 637 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..4c1183c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+*~
+*.pyc
+logactio.cfg
+build/
diff --git a/LogActio/Logger.py b/LogActio/Logger.py
new file mode 100644
index 0000000..692f36f
--- /dev/null
+++ b/LogActio/Logger.py
@@ -0,0 +1,75 @@
+#
+#   logactio  -  simple framework for doing configured action on certain
+#                log file events
+#
+#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import syslog
+
+LOGTYPE_FILE = 0
+LOGTYPE_SYSLOG = 1
+
+class FileLogger(object):
+    def __init__(self, logfile):
+        self.__logf = open(logfile, "a+")
+
+
+    def _log(self, lvl, msg):
+        self.__logf.write("[%i] %s\n" % (lvl, msg))
+        self.__logf.flush()
+
+
+    def _close(self):
+        self.__logf.close()
+
+
+
+class SysLogger(object):
+    def __init__(self, destname):
+        syslog.openlog(destname, syslog.LOG_PID|syslog.LOG_NOWAIT, syslog.LOG_USER)
+
+
+    def _log(self, lvl, msg):
+        syslog.syslog(syslog.LOG_NOTICE, msg)
+
+
+    def _close(self):
+        syslog.closelog()
+
+
+
+class Logger(object):
+    def __init__(self, logtype, logdest, verblvl):
+        self.__verblevel = verblvl
+        if logtype == LOGTYPE_FILE:
+            self.__logger = FileLogger(logdest)
+        elif logtype == LOGTYPE_SYSLOG:
+            self.__logger = SysLogger(logdest)
+
+    def Log(self, lvl, msg):
+        if lvl <= self.__verblevel:
+            self.__logger._log(lvl, msg)
+
+
+    def Close(self):
+        self.__logger._close()
diff --git a/LogActio/Message.py b/LogActio/Message.py
new file mode 100644
index 0000000..4ffb252
--- /dev/null
+++ b/LogActio/Message.py
@@ -0,0 +1,48 @@
+#
+#   logactio  -  simple framework for doing configured action on certain
+#                log file events
+#
+#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+MSG_SEND = 1
+MSG_SHUTDOWN = 2
+
+class Message(object):
+    def __init__(self, msgtype, prio, msg):
+        self.__msgtype = msgtype
+        self.__prio = prio
+        self.__msg = msg
+
+    def MessageType(self):
+        return self.__msgtype
+
+    def Priority(self):
+        return self.__prio;
+
+    def Message(self):
+        return self.__msg
+
+    def __str__(self):
+        return "[Type: %i, Prio: %i] %s" % (
+            self.__msgtype, self.__prio, self.__msg )
+
diff --git a/LogActio/ReporterQueue.py b/LogActio/ReporterQueue.py
new file mode 100644
index 0000000..1e87ece
--- /dev/null
+++ b/LogActio/ReporterQueue.py
@@ -0,0 +1,60 @@
+#
+#   logactio  -  simple framework for doing configured action on certain
+#                log file events
+#
+#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import threading, Queue
+import Message
+
+
+class ReporterQueue(object):
+    def __init__(self, qname, descr, processor):
+        self.__thread = None
+        self.__qname = qname
+        self.__description = descr
+        self.__queue = Queue.Queue()
+
+        self.__thread = threading.Thread(target=processor)
+
+
+    def GetName(self):
+        return self.__description
+
+
+    def _Start(self):
+        if not self.__thread.isAlive():
+            self.__thread.start()
+
+
+    def _Shutdown(self):
+        self.__queue.put(Message.Message(Message.MSG_SHUTDOWN, 0, None))
+        self.__queue.task_done()
+
+
+    def _QueueMsg(self, prio, msg):
+        self.__queue.put(Message.Message(Message.MSG_SEND, 0, msg))
+
+
+    def _QueueGet(self):
+        return self.__queue.get()
diff --git a/LogActio/Reporters/__init__.py b/LogActio/Reporters/__init__.py
new file mode 100644
index 0000000..8b6d11b
--- /dev/null
+++ b/LogActio/Reporters/__init__.py
@@ -0,0 +1,69 @@
+#
+#   logactio  -  simple framework for doing configured action on certain
+#                log file events
+#
+#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import sys
+from LogActio import Message, ReporterQueue
+
+
+class DefaultReporter(ReporterQueue.ReporterQueue):
+    def __init__(self, config, logger = None):
+        self.__alertprefix = config.has_key('prefix') and config['prefix'] or  "---> "
+        self.__log = logger
+        ReporterQueue.ReporterQueue.__init__(self,
+                                             "DefaultReporter",
+                                             "Default stdout reporter",
+                                             self.__processqueue)
+
+
+    def __processqueue(self):
+        done = False
+
+        # Process the message queue
+        while not done:
+            msg = self._QueueGet()
+
+            if( msg.MessageType() == Message.MSG_SHUTDOWN ):
+                # Prepare for shutdown
+                done = True
+
+            elif( msg.MessageType() == Message.MSG_SEND ):
+                if not self.__log:
+                    print "[DefaultReporter] %s" % msg
+                    sys.stdout.flush()
+                else:
+                    self.__log(0, "[DefaultReporter] %s" % msg)
+
+
+    def ProcessEvent(self, logfile, prefix, msg, count, threshold):
+        # Format the report message
+        if msg is None:
+            msg = "%s [%s] %s (%s)" % (self.__alertprefix, logfile, prefix, count)
+        else:
+            msg = "%s [%s] %s (%s): %s" % (self.__alertprefix, logfile, prefix, count, msg)
+
+        # Queue the message for sending
+        self._QueueMsg(0, msg)
+
diff --git a/LogActio/__init__.py b/LogActio/__init__.py
new file mode 100644
index 0000000..39bce93
--- /dev/null
+++ b/LogActio/__init__.py
@@ -0,0 +1,284 @@
+#
+#   logactio  -  simple framework for doing configured action on certain
+#                log file events
+#
+#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import sys, os, re, time, ConfigParser, threading, signal
+import ReporterQueue
+from LogActio.Reporters import DefaultReporter
+
+class WatcherThread(threading.Thread):
+    def __init__(self, logfile, polltime, reporter):
+        # This object will watch for changes in one particular log file
+        self.__rules = []
+        self.__logfile = logfile
+        self.__polltime = polltime is not None and int(polltime) or 30
+        self.__reporter = reporter
+        self.__shutdown = False
+        threading.Thread.__init__(self)
+
+
+    def GetLogfile(self):
+        return self.__logfile
+
+
+    def AddRule(self, prefix, regex, threshold):
+        # Adds a rule specific for this log file
+        rule = {"prefix": prefix,
+                "regex": re.compile(regex),
+                "threshold": int(threshold),
+                "current_count": 0,
+                "alerts_sent": 0}
+        self.__rules.append(rule)
+
+    def StartWatcher(self):
+        # Start the reporter module
+        self.__reporter._Start()
+        # Start the thread with this watcher
+        self.start()
+
+    def run(self):
+        # This is started by threading.Thread
+        try:
+            fp = fp = open(self.__logfile, "r")
+            fp.seek(0, 2)
+        except IOError:
+            self.Shutdown()
+            raise Exception("Could not access logfile: %s" % self.__logfile)
+
+        # Whenever the file changes, we receive the lines here
+        while not self.__shutdown:
+            where = fp.tell()
+            line = fp.readline()
+            if len(line) == 0:
+                time.sleep(self.__polltime)
+                fp.seek(where)
+                continue
+
+            for alert in self.__rules:
+                m = alert["regex"].match(line.splitlines()[0])
+                # If the received log line matches the regex
+                if not self.__shutdown and m:
+                    alert["current_count"] += 1
+
+                    # If the threshold has been reached, report the incident
+                    if alert["threshold"] == 0 or (alert["current_count"] % alert["threshold"] == 0):
+                        alert["alerts_sent"] += 1
+                        info = "|".join(m.groups()) # Gather regex exctracted info
+                        if len(info) == 0:
+                            info = None
+                        self.__reporter.ProcessEvent(self.__logfile, alert["prefix"], info,
+                                                     alert["current_count"], alert["threshold"])
+        fp.close()
+        return 0
+
+
+    def Shutdown(self):
+        self.__shutdown = True
+        self.__reporter._Shutdown()
+
+
+class LogActio(object):
+    def __init__(self, cfgfile, daemon=False, pidfile=None, logger=None, stdout="/dev/null"):
+        try:
+            self.__cfg = ConfigParser.ConfigParser()
+            res = self.__cfg.read(cfgfile)
+            if len(res) == 0:
+                raise Exception("Could not load the configuration file '%s'" % cfgfile)
+        except Exception, e:
+            raise e
+
+        self.__watchthreads = []
+        self.__shutdown = False
+        self.__daemon = daemon
+        self.__pidfp = None
+        self.__pidfilename = pidfile
+        self.__devnull = None
+
+        if logger is not None:
+            self.__log = logger
+        else:
+            self.__log = self.__logfnc
+
+        if daemon:
+            self.__daemonise(stdout)
+
+        if self.__pidfilename:
+            self.__pidfp = os.open(self.__pidfilename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0600)
+            os.write(self.__pidfp, "%i" % os.getpid())
+
+        self.__parse_cfg()
+
+
+    def __logfnc(self, lvl, msg):
+        print "[%i] %s" % (lvl, msg)
+
+
+    def __daemonise(self, redir_stdout):
+        try:
+            pid = os.fork()
+            if pid > 0:
+                sys.exit(0)
+        except OSError, e:
+            sys.stderr.write("Failed to daemonise [step 1]: %s\n" % str(e))
+            sys.exit(1)
+
+        os.chdir("/")
+        os.setsid()
+        os.umask(0)
+
+        try:
+            pid = os.fork()
+            if pid > 0:
+                sys.exit(0)
+        except OSError, e:
+            sys.stderr.write("Failed to daemonise [step 2]: %s\n" % str(e))
+            sys.exit(1)
+
+        stdin = file("/dev/null", "r")
+        stdout = file(redir_stdout, "a+", 0)
+        stderr = file("/dev/null", "a+", 0)
+        os.dup2(stdin.fileno(), sys.stdin.fileno())
+        os.dup2(stdout.fileno(), sys.stdout.fileno())
+        os.dup2(stderr.fileno(), sys.stderr.fileno())
+
+        self.__log(0, "Daemonised logactio, pid %i" % os.getpid())
+
+
+    def __parse_cfg(self):
+        __logfileidx = []
+        __reporters = {}
+
+        # Retrieve all configured reporter modules, sections prefixed with 'Reporter:'
+        repre = re.compile("Reporter:(.*)", re.IGNORECASE)
+        for entry in [rr for rr in self.__cfg.sections() if repre.match(rr)]:
+            repname = repre.match(entry).groups()[0]
+
+            if __reporters.has_key(repname):
+                raise Exception("The %s reporter has already been configured" % repname)
+
+            # Get configuration for reporter
+            repcfg = {}
+            for k,v in self.__cfg.items(entry):
+                repcfg[k] = v
+
+            if repname == "Default":
+                __reporters[repname] = DefaultReporter(repcfg. self.__log)
+            else:
+                raise Exception("The module loader for reporters have not been implemented (%s)" % repname)
+            del repcfg
+            self.__log(2, "Configured reporter %s: %s" % (repname, __reporters[repname].GetName()))
+
+        if not __reporters.has_key("Default"):
+            __reporters["Default"] = DefaultReporter({}, self.__log)
+
+        self.__log(1, "Available reporter modules: %s" % ", ".join(__reporters.keys()))
+
+        # Retrieve all logfile configurations, sections prefixed with 'Logfile:'
+        lfre = re.compile("Logfile:(.*)", re.IGNORECASE)
+        for entry in [lf for lf in self.__cfg.sections() if lfre.match(lf)]:
+            logname = lfre.match(entry).groups()[0]
+            try:
+                # Find the saved array index for this logfile
+                logfile = self.__cfg.get(entry, "file")
+                idx = __logfileidx.index(logname)
+            except ValueError:
+                # If index was not found, it's a new logfile
+                __logfileidx.append(logname)
+                idx = __logfileidx.index(logname)
+
+                # Extract the poll time
+                try:
+                    polltime = self.__cfg.get(entry, "polltime")
+                except ConfigParser.NoOptionError:
+                    polltime = None
+
+                # Extract the reporter to use for this log file
+                try:
+                    repname = self.__cfg.get(entry, "reporter")
+                    reporter = __reporters[repname]
+                except ConfigParser.NoOptionError:
+                    reporter = __reporters["Default"]
+
+                # Create a new thread which will watch this particular log file
+                # and will use the configured reporter to handle matching events
+                self.__watchthreads.append(WatcherThread(logfile, polltime, reporter))
+                self.__log(3, "Prepared [%s]: %s (%s) => %s" % (
+                        logname, logfile, polltime, reporter.GetName()))
+
+        self.__log(1, "Configured log files: %s" % ", ".join(__logfileidx))
+
+        # Retrieve all configured rules
+        rulsre = re.compile("Rule:(.*)", re.IGNORECASE)
+        for entry in [rl for rl in self.__cfg.sections() if rulsre.match(rl)]:
+            rulename = rulsre.match(entry).groups()[0]
+            try:
+                # Look up the WatchThread index for the logfile this rule requires
+                logf = self.__cfg.get(entry, "logfile")
+                idx = __logfileidx.index(logf)
+            except ValueError:
+                raise Exception("** ERROR ** Logfile '%s' is not configured" % self.__cfg.get(entry, "logfile"))
+            # Add the rule to the proper WatchThread
+            self.__watchthreads[idx].AddRule(rulename,
+                                         self.__cfg.get(entry, "regex"),
+                                         self.__cfg.get(entry, "threshold"))
+        del __logfileidx
+
+
+
+    def __Shutdown(self):
+        for wt in self.__watchthreads:
+            wt.Shutdown()
+            wt.join()
+            self.__shutdown = True
+
+        if self.__pidfilename and self.__pidfp:
+            os.close(self.__pidfp)
+            os.unlink(self.__pidfilename)
+
+
+    def __sighandler(self, signum, frame):
+        if signum == signal.SIGINT or signum == signal.SIGTERM or signum == signal.SIGUSR1:
+            if self.__shutdown:
+                self.__log(1, "Shutdown already in progress")
+                return
+
+            self.__log(0, "logactio shutdown initiated")
+            self.__Shutdown()
+
+
+    def Run(self):
+        i = 0
+
+        signal.signal(signal.SIGINT, self.__sighandler)
+        signal.signal(signal.SIGTERM, self.__sighandler)
+        signal.signal(signal.SIGUSR1, self.__sighandler)
+        for wt in self.__watchthreads:
+            wt.StartWatcher()
+            i += 1
+
+        while not self.__shutdown:
+            time.sleep(5)
+
+        self.__log(0, "logactio stopped")
diff --git a/logactio b/logactio
new file mode 100755
index 0000000..d56bf70
--- /dev/null
+++ b/logactio
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+#
+#   logactio  -  simple framework for doing configured action on certain
+#                log file events
+#
+#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import sys, optparse
+from LogActio import LogActio, Logger
+
+
+if __name__ == '__main__':
+    parser = optparse.OptionParser()
+
+    parser.add_option("-d", "--daemon", action="store_true", default=False,
+                      dest="daemon",
+                      help="Run as a daemon")
+    parser.add_option("-p", "--pid-file",
+                      dest="pidfile", metavar="PID-FILE", default=None,
+                      help="Put pid file of logactio in this file")
+    parser.add_option("--stdout-redir",
+                      dest="stdoutredir", metavar="FILE", default="/dev/null",
+                      help="Redirect all stdout data to this file (only active when running as daemon)")
+    parser.add_option("-c", "--config",
+                      dest="cfg", metavar="FILE", default="/etc/logactio.cfg",
+                      help="Configuration file for logactio (Default: %default)")
+    parser.add_option("-v", "--verbose", action="count", default=0,
+                      dest="verb",
+                      help="Increase the log verbosity")
+    parser.add_option("-L", "--log-type", type="choice", choices=["syslog","file"],
+                      dest="logtype", default="syslog",
+                      help="Should logging go to file or syslog? (default: %default)")
+    parser.add_option("-l", "--log-file",
+                      dest="logfile", metavar="LOG-FILE",
+                      help="Filename of the log file to use")
+    parser.add_option("-s", "--syslog-id",
+                      dest="syslogid", metavar="SYSLOG-NAME", default="logactio",
+                      help="syslog ident to use for syslog events")
+    parser.add_option("--trace", action="store_true",
+                      dest="trace", default=False,
+                      help="On errors, show a backtrace")
+    (opts, args) = parser.parse_args()
+
+    try:
+        if opts.logtype == "syslog":
+            logger = Logger.Logger(Logger.LOGTYPE_SYSLOG, opts.syslogid, opts.verb)
+        elif opts.logtype == "file":
+            logger = Logger.Logger(Logger.LOGTYPE_FILE, opts.logfile, opts.verb)
+        else:
+            raise Exception("Unknown log type '%s'" % opts.logtype)
+
+        main = LogActio(opts.cfg, opts.daemon, opts.pidfile, logger.Log, opts.stdoutredir)
+        main.Run()
+        sys.exit(0)
+    except Exception, e:
+        if opts.trace:
+            import traceback
+            traceback.print_exc(file=sys.stdout)
+        else:
+            print "** ERROR ** %s" % str(e)
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..f367c45
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python2
+
+from distutils.core import setup
+import commands, sys
+
+version = "0.01"
+
+setup(name="libactio",
+      version=version,
+      description="Simple framework for executing actions on certain log events",
+      author="David Sommerseth",
+      author_email="dazo@users.sourceforge.net",
+      url="",
+      license="GPLv2 only",
+      packages=["LogActio"],
+      scripts=["logactio"]
+)