path: root/LogActio/__init__.py

#
#   logactio  -  simple framework for doing configured action on certain
#                log file events
#
#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
#
#   For the avoidance of doubt the "preferred form" of this code is one which
#   is in an open unpatent encumbered format. Where cryptographic key signing
#   forms part of the process of creating an executable the information
#   including keys needed to generate an equivalently functional executable
#   are deemed to be part of the source code.
#

import sys, os, re, time, ConfigParser, threading, signal
import ReporterQueue
from LogActio.Reporters import DefaultReporter

class WatcherThread(threading.Thread):
    def __init__(self, logfile, polltime, reporters):
        # This object will watch for changes in one particular log file
        self.__rules = []
        self.__logfile = logfile
        self.__polltime = polltime is not None and int(polltime) or 30
        self.__reporters = reporters
        self.__shutdown = False
        threading.Thread.__init__(self)


    def GetLogfile(self):
        return self.__logfile


    def AddRule(self, prefix, regex, threshold, timeframe, reporters):
        # Adds a rule specific for this log file
        rule = {"prefix": prefix,
                "regex": re.compile(regex),
                "threshold": int(threshold),
                "timeframe": timeframe and int(timeframe) or None,
                "lastseen": 0,
                "current_count": 0,
                "alerts_sent": 0,
                "reporters": reporters}
        self.__rules.append(rule)


    def StartWatcher(self):
        # Start the default reporter modules
        for rep in self.__reporters:
            rep._Start()

        # Start reporter modules declared in rules
        for r in self.__rules:
            if r.has_key("reporters") and r["reporters"] is not None:
               for rep in  r["reporters"]:
                   rep._Start()

        # Start the thread with this watcher
        self.start()

    def run(self):
        # This is started by threading.Thread
        try:
            fp = fp = open(self.__logfile, "r")
            fp.seek(0, 2)
        except IOError:
            self.Shutdown()
            raise Exception("Could not access logfile: %s" % self.__logfile)

        # Whenever the file changes, we receive the lines here
        while not self.__shutdown:
            where = fp.tell()
            line = fp.readline()
            if len(line) == 0:
                time.sleep(self.__polltime)
                fp.seek(where)
                continue

            now = int(time.time())
            for alert in self.__rules:
                m = alert["regex"].match(line.splitlines()[0])
                # If the received log line matches the regex
                if not self.__shutdown and m:
                    alert["current_count"] += 1

                    # If the threshold has been reached and within the given time frame,
                    # report the incident
                    if (alert["threshold"] == 0
                        or ((alert["current_count"] % alert["threshold"] == 0)
                            and (alert["timeframe"] is None
                                 or now <= (alert["lastseen"] + alert["timeframe"])))):
                        alert["alerts_sent"] += 1
                        info = "|".join(m.groups()) # Gather regex exctracted info
                        if len(info) == 0:
                            info = None

                        # Send the alert event to be processed, prioritise the
                        # rule specific reporter over the default reporter
                        rep = alert.has_key("reporters") and alert["reporters"] or self.__reporters
                        for r in rep:
                            r.ProcessEvent(self.__logfile, alert["prefix"], info,
                                           alert["current_count"], alert["threshold"])
                        alert["lastseen"] = 0
                        continue

                    if (alert["timeframe"] and (alert["lastseen"] > 0)
                        and (now >= (alert["lastseen"] + alert["timeframe"]))):
                        # If the time-frame have timed out, reset it
                        alert["lastseen"] = 0
                    else:
                        alert["lastseen"] = now

        fp.close()
        return 0


    def Shutdown(self):
        self.__shutdown = True

        # Shutdown rule specific reporters
        for r in self.__rules:
            if r.has_key("reporters") and r["reporters"] is not None:
                for rep in r["reporters"]:
                    rep._Shutdown()

        # Shutdown default reporters
        for rep in self.__reporters:
            rep._Shutdown()


class LogActio(object):
    def __init__(self, cfgfile, daemon=False, pidfile=None, logger=None, stdout="/dev/null"):
        try:
            self.__cfg = ConfigParser.ConfigParser()
            res = self.__cfg.read(cfgfile)
            if len(res) == 0:
                raise Exception("Could not load the configuration file '%s'" % cfgfile)
        except Exception, e:
            raise e

        self.__watchthreads = []
        self.__ext_modules = {}
        self.__shutdown = False
        self.__daemon = daemon
        self.__pidfp = None
        self.__pidfilename = pidfile
        self.__devnull = None

        if logger is not None:
            self.__log = logger
        else:
            self.__log = self.__logfnc

        if daemon:
            self.__daemonise(stdout)

        if self.__pidfilename:
            self.__pidfp = os.open(self.__pidfilename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0600)
            os.write(self.__pidfp, "%i" % os.getpid())

        self.__parse_cfg()


    def __logfnc(self, lvl, msg):
        print "[%i] %s" % (lvl, msg)


    def __daemonise(self, redir_stdout):
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write("Failed to daemonise [step 1]: %s\n" % str(e))
            sys.exit(1)

        os.chdir("/")
        os.setsid()
        os.umask(0)

        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write("Failed to daemonise [step 2]: %s\n" % str(e))
            sys.exit(1)

        stdin = file("/dev/null", "r")
        stdout = file(redir_stdout, "a+", 0)
        stderr = file("/dev/null", "a+", 0)
        os.dup2(stdin.fileno(), sys.stdin.fileno())
        os.dup2(stdout.fileno(), sys.stdout.fileno())
        os.dup2(stderr.fileno(), sys.stderr.fileno())

        self.__log(0, "Daemonised logactio, pid %i" % os.getpid())


    def __parse_cfg(self):
        __logfileidx = []
        __reporters = {}

        # Retrieve all configured reporter modules, sections prefixed with 'Reporter:'
        repre = re.compile("Reporter:(.*)", re.IGNORECASE)
        for entry in [rr for rr in self.__cfg.sections() if repre.match(rr)]:
            repname = repre.match(entry).groups()[0]

            if __reporters.has_key(repname):
                raise Exception("The %s reporter has already been configured" % repname)

            # Get configuration for reporter
            repcfg = {}
            for k,v in self.__cfg.items(entry):
                repcfg[k] = v

            if repcfg.has_key("module"):
                # import this reporter module, but only if it is unkown
                extmodule = repcfg["module"]
                if not self.__ext_modules.has_key(extmodule):
                    self.__ext_modules[extmodule] = __import__("LogActio.Reporters.%s" % extmodule,
                                                               fromlist="LogActio.Reporters")
            else:
                extmodule = None

            if repname == "Default":
                __reporters[repname] = DefaultReporter(repcfg. self.__log)
            elif extmodule is not None:
                __reporters[repname] = self.__ext_modules[extmodule].InitReporter(repcfg, self.__log)
            else:
                raise Exception("Dazed and confused - this shouldn't happen (repname: %s)" % repname)

            del repcfg
            self.__log(2, "Configured reporter %s: %s" % (repname, __reporters[repname].GetName()))

        if not __reporters.has_key("Default"):
            __reporters["Default"] = DefaultReporter({}, self.__log)

        self.__log(1, "Available reporter modules: %s" % ", ".join(__reporters.keys()))

        # Retrieve all logfile configurations, sections prefixed with 'Logfile:'
        lfre = re.compile("Logfile:(.*)", re.IGNORECASE)
        for entry in [lf for lf in self.__cfg.sections() if lfre.match(lf)]:
            logname = lfre.match(entry).groups()[0]
            try:
                # Find the saved array index for this logfile
                logfile = self.__cfg.get(entry, "file")
                idx = __logfileidx.index(logname)
            except ValueError:
                # If index was not found, it's a new logfile
                __logfileidx.append(logname)
                idx = __logfileidx.index(logname)

                # Extract the poll time
                try:
                    polltime = self.__cfg.get(entry, "polltime")
                except ConfigParser.NoOptionError:
                    polltime = None

                # Extract the default reporters to use for this log file
                try:
                    defreps = []
                    repnames = self.__cfg.get(entry, "reporters")
                    for repname in [n.strip() for n in repnames.split(",")]:
                        defreps.append(__reporters[repname])
                except ConfigParser.NoOptionError:
                    depreps = [__reporters["Default"]]
                except KeyError, e:
                    raise Exception("No reporters are configured as '%s'" % repname)

                # Create a new thread which will watch this particular log file
                # and will use the configured reporter to handle matching events
                self.__watchthreads.append(WatcherThread(logfile, polltime, defreps))
                self.__log(3, "Prepared [%s]: %s (%s) => %s" % (
                        logname, logfile, polltime, ", ".join([r.GetName() for r in defreps])))

        self.__log(1, "Configured log files: %s" % ", ".join(__logfileidx))

        # Retrieve all configured rules
        rulsre = re.compile("Rule:(.*)", re.IGNORECASE)
        for entry in [rl for rl in self.__cfg.sections() if rulsre.match(rl)]:
            rulename = rulsre.match(entry).groups()[0]
            try:
                # Look up the WatchThread index for the logfile this rule requires
                logf = self.__cfg.get(entry, "logfile")
                idx = __logfileidx.index(logf)
            except ValueError:
                raise Exception("** ERROR ** Logfile '%s' is not configured" % self.__cfg.get(entry, "logfile"))
            try:
                # Check if this rule has specific reporters confingured
                rulereps = []
                repnames = self.__cfg.get(entry, "reporters")
                for repname in [n.strip() for n in repnames.split(",")]:
                    rulereps.append(__reporters[repname])
            except ConfigParser.NoOptionError:
                # If nothing was found, that's okay
                reps = None
            except KeyError, e:
                raise Exception("No reporters are configured as '%s'" % repname)

            # Add the rule to the proper WatchThread
            self.__watchthreads[idx].AddRule(rulename,
                                             self.__cfg.get(entry, "regex"),
                                             self.__cfg.get(entry, "threshold"),
                                             (self.__cfg.has_option(entry, "time-frame")
                                              and self.__cfg.get(entry, "time-frame") or None),
                                             rulereps)
            if rulereps is not None and len(rulereps) > 0:
                self.__log(3, "Rule reporters prepared: [%s] => %s" %
                           (rulename, ", ".join([r.GetName() for r in rulereps])))
        del __logfileidx



    def __Shutdown(self):
        for wt in self.__watchthreads:
            wt.Shutdown()
            wt.join()
            self.__shutdown = True

        if self.__pidfilename and self.__pidfp:
            os.close(self.__pidfp)
            os.unlink(self.__pidfilename)


    def __sighandler(self, signum, frame):
        if signum == signal.SIGINT or signum == signal.SIGTERM or signum == signal.SIGUSR1:
            if self.__shutdown:
                self.__log(1, "Shutdown already in progress")
                return

            self.__log(0, "logactio shutdown initiated")
            self.__Shutdown()


    def Run(self):
        i = 0

        signal.signal(signal.SIGINT, self.__sighandler)
        signal.signal(signal.SIGTERM, self.__sighandler)
        signal.signal(signal.SIGUSR1, self.__sighandler)
        for wt in self.__watchthreads:
            wt.StartWatcher()
            i += 1

        while not self.__shutdown:
            time.sleep(5)

        self.__log(0, "logactio stopped")