Added a new threshold parameter: threshold-type

This can be set to either 'rule' or 'exact'.  If not defined,
it defaults to 'rule' which is exactly the same as before.

In 'rule' mode, the threshould counter is increased each time
the regular expression triggers a match.

By switching to 'exact', it will be defined a threshold counter
based on the conntents of the regex groups when a match is found.
This gives a more fine grained threshold counter, which can be
used for example for blocking specific IP addresses after a
certain number of failed attempts is caught.

Signed-off-by: David Sommerseth <dazo@users.sourceforge.net>

3 files changed, 205 insertions, 32 deletions

diff --git a/LogActio/ThresholdWatch.py b/LogActio/ThresholdWatch.py
new file mode 100644
index 0000000..033109c
--- /dev/null
+++ b/LogActio/ThresholdWatch.py
@@ -0,0 +1,173 @@
+#
+#   logactio  -  simple framework for doing configured action on certain
+#                log file events
+#
+#   Copyright 2013   David Sommerseth <dazo@users.sourceforge.net>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import hashlib, time
+
+
+class ThresholdType_Rule(object):
+    def __init__(self, params):
+        if not params.has_key("threshold"):
+            raise ValueError("Missing required 'threshold' parameter")
+
+        self.__threshold = int(params["threshold"])
+        self.__timeframe = params["timeframe"] and int(params["timeframe"]) or None
+        self.__ratelimit = params["ratelimit"] and int(params["ratelimit"]) or None
+        self.__lastseen = 0
+        self.__lastsent = 0
+        self.__currentcount = 0
+
+    def CheckThreshold(self, alert, notused_regexmatch):
+        now = int(time.time())
+        self.__currentcount += 1
+        ret = (self.__threshold == 0
+                or ((self.__currentcount % self.__threshold == 0)
+                    and (self.__timeframe is None
+                         or now <= (self.__lastseen + self.__timeframe)))
+                and (self.__ratelimit is None or now > (self.__lastsent + self.__ratelimit)))
+
+        if (self.__timeframe and (self.__lastseen > 0)
+            and (now >= (self.__lastseen + self.__timeframe))):
+            # If the time-frame have timed out, reset it
+            self.__lastseen = 0
+        else:
+            self.__lastseen = now
+
+        return ret
+
+
+    def ClearTimeTrackers(self, notused_regexmatch):
+        self.__lastseen = 0
+        self.__lastsent = 0
+
+
+    def GetThreshold(self):
+        return self.__threshold
+
+
+    def GetCurrentCount(self, rgmatch):
+        return self.__currentcount
+
+
+
+class ThresholdType_Exact(object):
+    def __init__(self, params):
+        self.__matchedgroups = {}
+
+        self.__threshold = int(params["threshold"])
+        self.__timeframe = params["timeframe"] and int(params["timeframe"]) or None
+        self.__ratelimit = params["ratelimit"] and int(params["ratelimit"]) or None
+
+
+    def __gen_hash(self, regexmatch):
+        return hashlib.sha384("|".join(regexmatch)).hexdigest()
+
+
+    def CheckThreshold(self, alert, regexmatch):
+        now = int(time.time())
+        ret = False
+
+        # If threshold is 0 or 1, then we process all records
+        if self.__threshold < 2:
+            return True
+
+        # Check if we have a regexmatch on this from earlier checks
+        rghash = self.__gen_hash(regexmatch)
+        if self.__matchedgroups.has_key(rghash):
+            lastevent = self.__matchedgroups[rghash]
+            lastevent["count"] += 1
+
+            ret = ((lastevent["count"] % self.__threshold == 0)
+                   and (self.__timeframe is None
+                        or now <= (lastevent["lastseen"] + self.__timeframe))
+                   and (self.__ratelimit is None or now > (lastevent["lastsent"] + self.__ratelimit)))
+
+            if (self.__timeframe and (lastevent["lastseen"] > 0)
+                and (now >= (lastevent["lasteen"] + self.__timeframe))):
+                # If the time-frame have timed out, reset it
+                self.__lastseen = 0
+            else:
+                self.__lastseen = now
+        else:
+            # Not seen before, register it as a new one
+            self.__matchedgroups[rghash] = {
+                "count": 1,
+                "lastseen": now,
+                "lastsent": 0
+                }
+            ret = (1 % self.__threshold == 0)
+
+        return ret
+
+
+    def ClearTimeTrackers(self, regexmatch):
+        rghash = self.__gen_hash(regexmatch)
+        if self.__matchedgroups.has_key(rghash):
+            self.__matchedgroups[rghash]["lasteen"] = 0
+            self.__matchedgroups[rghash]["lastsent"] = 0
+
+
+    def GetThreshold(self):
+        return self.__threshold
+
+
+    def GetCurrentCount(self, regexmatch):
+        rghash = self.__gen_hash(regexmatch)
+        if self.__matchedgroups.has_key(rghash):
+            return self.__matchedgroups[rghash]["count"]
+        else:
+            return 0
+
+
+
+class ThresholdWatch(object):
+    WATCHTYPE_RULE = 1
+    WATCHTYPE_EXACT = 2
+
+    def __init__(self, watchtype, params):
+        self.__watchtype = watchtype
+        if watchtype == self.WATCHTYPE_RULE:
+            self.__thresholdtype = ThresholdType_Rule(params)
+        elif watchtype == self.WATCHTYPE_EXACT:
+            self.__thresholdtype = ThresholdType_Exact(params)
+        else:
+            raise ValueError("Invalid watchtype parameter")
+
+
+    def CheckThreshold(self, alert, regexmatch):
+        return self.__thresholdtype.CheckThreshold(alert, regexmatch)
+
+
+    def ClearTimeTrackers(self, regexmatch):
+        self.__thresholdtype.ClearTimeTrackers(regexmatch)
+
+
+    def GetThreshold(self):
+        return self.__thresholdtype.GetThreshold()
+
+
+    def GetCurrentCount(self, regexmatch):
+        return self.__thresholdtype.GetCurrentCount(regexmatch)
+
diff --git a/LogActio/__init__.py b/LogActio/__init__.py
index 5a7c167..a7f0c54 100644
--- a/LogActio/__init__.py
+++ b/LogActio/__init__.py
@@ -2,7 +2,7 @@
 #   logactio  -  simple framework for doing configured action on certain
 #                log file events
 #
-#   Copyright 2012   David Sommerseth <dazo@users.sourceforge.net>
+#   Copyright 2012 - 2013   David Sommerseth <dazo@users.sourceforge.net>
 #
 #   This program is free software; you can redistribute it and/or modify
 #   it under the terms of the GNU General Public License as published by
@@ -26,6 +26,7 @@
 
 import sys, os, re, time, ConfigParser, threading, signal
 import ReporterQueue
+from LogActio.ThresholdWatch import ThresholdWatch
 from LogActio.Reporters import DefaultReporter
 
 class WatcherThread(threading.Thread):
@@ -43,18 +44,22 @@ class WatcherThread(threading.Thread):
         return self.__logfile
 
 
-    def AddRule(self, prefix, regex, threshold, timeframe, ratelimit, resetrules, reporters):
+    def AddRule(self, prefix, regex, thrtype, threshold, timeframe, ratelimit, resetrules, reporters):
+        # Convert threshold type from string to known internal variables
+        if thrtype is None or thrtype.lower() == "rule":
+            thrtype = ThresholdWatch.WATCHTYPE_RULE
+        elif thrtype.lower() == "exact":
+            thrtype = ThresholdWatch.WATCHTYPE_EXACT
+
         # Adds a rule specific for this log file
         rule = {"prefix": prefix,
                 "regex": re.compile(regex),
-                "threshold": int(threshold),
-                "timeframe": timeframe and int(timeframe) or None,
-                "ratelimit": ratelimit and int(ratelimit) or None,
+                "threshold": ThresholdWatch(thrtype,
+                                            {"threshold": threshold,
+                                             "timeframe": timeframe,
+                                             "ratelimit": ratelimit}),
                 "resetrules": resetrules,
-                "lastseen": 0,
-                "current_count": 0,
                 "alerts_sent": 0,
-                "lastsent": 0,
                 "reporters": reporters}
         self.__rules.append(rule)
 
@@ -105,25 +110,18 @@ class WatcherThread(threading.Thread):
                     fp.seek(where)
                 continue
 
-            now = int(time.time())
             resetlist = []
             for alert in self.__rules:
                 m = alert["regex"].match(line.splitlines()[0])
                 # If the received log line matches the regex
                 if not self.__shutdown and m:
-                    alert["current_count"] += 1
-
+                    regexmatch = m.groups()
                     # If the threshold has been reached and within the given time frame,
                     # report the incident.  Also, if we have an rate-limit, only send
                     # a report it is 'rate-limit seconds' since last report.
-                    if (alert["threshold"] == 0
-                        or ((alert["current_count"] % alert["threshold"] == 0)
-                            and (alert["timeframe"] is None
-                                 or now <= (alert["lastseen"] + alert["timeframe"])))
-                        and (alert["ratelimit"] is None or now > (alert["lastsent"] + alert["ratelimit"]))):
+                    if alert["threshold"].CheckThreshold(alert, regexmatch):
                         alert["alerts_sent"] += 1
-                        alert["lastsent"] = now
-                        info = "|".join(m.groups()) # Gather regex exctracted info
+                        info = "|".join(regexmatch) # Gather regex exctracted info
                         if len(info) == 0:
                             info = None
 
@@ -132,31 +130,22 @@ class WatcherThread(threading.Thread):
                         rep = alert.has_key("reporters") and alert["reporters"] or self.__reporters
                         for r in rep:
                             r.ProcessEvent(self.__logfile, alert["prefix"], info,
-                                           alert["current_count"], alert["threshold"])
+                                           alert["threshold"].GetCurrentCount(regexmatch), alert["threshold"].GetThreshold())
 
                         # If reset-rule-rate-limits is set, make a note to reset these
                         # counters after all alerts have been processed
                         if alert["resetrules"]:
                             for r in alert["resetrules"]:
-                                resetlist.append(r)
+                                resetlist.append((r, regexmatch))
 
-                        alert["lastseen"] = 0
                         continue
 
-                    if (alert["timeframe"] and (alert["lastseen"] > 0)
-                        and (now >= (alert["lastseen"] + alert["timeframe"]))):
-                        # If the time-frame have timed out, reset it
-                        alert["lastseen"] = 0
-                    else:
-                        alert["lastseen"] = now
-
             # If we have some reset tasks scheduled, perform them now
             for reset in resetlist:
-                for rule in self.__rules:
+                for (rule, rgmatch) in self.__rules:
                     # Reset the lastsent and lastseen flags for the given rules
                     if rule["prefix"] == reset:
-                        rule["lastsent"] = 0
-                        rule["lastseen"] = 0
+                        rule["threshold"].ClearTimeTrackers(rgmatch)
 
         fp.close()
         return 0
@@ -349,6 +338,8 @@ class LogActio(object):
             # Add the rule to the proper WatchThread
             self.__watchthreads[idx].AddRule(rulename,
                                              self.__cfg.get(entry, "regex"),
+                                             (self.__cfg.has_option(entry, "threshold-type")
+                                              and self.__cfg.get(entry, "threshold-type") or None),
                                              self.__cfg.get(entry, "threshold"),
                                              (self.__cfg.has_option(entry, "time-frame")
                                               and self.__cfg.get(entry, "time-frame") or None),
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
index 9ee3cb3..6353142 100644
--- a/docs/source/configuration.rst
+++ b/docs/source/configuration.rst
@@ -1,4 +1,4 @@
-.. Copyright 2012 David Sommerseth <dazo@users.sourceforge.net>
+.. Copyright 2012 - 2013 David Sommerseth <dazo@users.sourceforge.net>
 
    This is free software: you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
@@ -330,6 +330,15 @@ A Rules section consists of two required configuration variables:
   This sets how many times this event should match before triggering the
   reporter.
 
+* threshold-type:
+
+  This defines how the threshold counter works.  By default, it is set to
+  *rule*.  This will increase the "hit counter" each time this watch rule
+  is triggered.  By setting the *threshold-type* parameter to *exact* it
+  will also consider the regex groups defined in the *regex* parameter.
+  When using the *exact* type, it will have individual threshold counters
+  per group contents for each time this rule is triggered.
+
 The optional settings are:
 
 * reporters