From 3b96eeb9a932b55c9ef13f5a9649c30bde86ac14 Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Thu, 26 Dec 2013 17:37:01 +0100 Subject: Added a new threshold parameter: threshold-type This can be set to either 'rule' or 'exact'. If not defined, it defaults to 'rule' which is exactly the same as before. In 'rule' mode, the threshould counter is increased each time the regular expression triggers a match. By switching to 'exact', it will be defined a threshold counter based on the conntents of the regex groups when a match is found. This gives a more fine grained threshold counter, which can be used for example for blocking specific IP addresses after a certain number of failed attempts is caught. Signed-off-by: David Sommerseth --- LogActio/ThresholdWatch.py | 173 +++++++++++++++++++++++++++++++++++++++++++++ LogActio/__init__.py | 53 ++++++-------- 2 files changed, 195 insertions(+), 31 deletions(-) create mode 100644 LogActio/ThresholdWatch.py (limited to 'LogActio') diff --git a/LogActio/ThresholdWatch.py b/LogActio/ThresholdWatch.py new file mode 100644 index 0000000..033109c --- /dev/null +++ b/LogActio/ThresholdWatch.py @@ -0,0 +1,173 @@ +# +# logactio - simple framework for doing configured action on certain +# log file events +# +# Copyright 2013 David Sommerseth +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# For the avoidance of doubt the "preferred form" of this code is one which +# is in an open unpatent encumbered format. Where cryptographic key signing +# forms part of the process of creating an executable the information +# including keys needed to generate an equivalently functional executable +# are deemed to be part of the source code. +# + +import hashlib, time + + +class ThresholdType_Rule(object): + def __init__(self, params): + if not params.has_key("threshold"): + raise ValueError("Missing required 'threshold' parameter") + + self.__threshold = int(params["threshold"]) + self.__timeframe = params["timeframe"] and int(params["timeframe"]) or None + self.__ratelimit = params["ratelimit"] and int(params["ratelimit"]) or None + self.__lastseen = 0 + self.__lastsent = 0 + self.__currentcount = 0 + + def CheckThreshold(self, alert, notused_regexmatch): + now = int(time.time()) + self.__currentcount += 1 + ret = (self.__threshold == 0 + or ((self.__currentcount % self.__threshold == 0) + and (self.__timeframe is None + or now <= (self.__lastseen + self.__timeframe))) + and (self.__ratelimit is None or now > (self.__lastsent + self.__ratelimit))) + + if (self.__timeframe and (self.__lastseen > 0) + and (now >= (self.__lastseen + self.__timeframe))): + # If the time-frame have timed out, reset it + self.__lastseen = 0 + else: + self.__lastseen = now + + return ret + + + def ClearTimeTrackers(self, notused_regexmatch): + self.__lastseen = 0 + self.__lastsent = 0 + + + def GetThreshold(self): + return self.__threshold + + + def GetCurrentCount(self, rgmatch): + return self.__currentcount + + + +class ThresholdType_Exact(object): + def __init__(self, params): + self.__matchedgroups = {} + + self.__threshold = int(params["threshold"]) + self.__timeframe = params["timeframe"] and int(params["timeframe"]) or None + self.__ratelimit = params["ratelimit"] and int(params["ratelimit"]) or None + + + def __gen_hash(self, regexmatch): + return hashlib.sha384("|".join(regexmatch)).hexdigest() + + + def CheckThreshold(self, alert, regexmatch): + now = int(time.time()) + ret = False + + # If threshold is 0 or 1, then we process all records + if self.__threshold < 2: + return True + + # Check if we have a regexmatch on this from earlier checks + rghash = self.__gen_hash(regexmatch) + if self.__matchedgroups.has_key(rghash): + lastevent = self.__matchedgroups[rghash] + lastevent["count"] += 1 + + ret = ((lastevent["count"] % self.__threshold == 0) + and (self.__timeframe is None + or now <= (lastevent["lastseen"] + self.__timeframe)) + and (self.__ratelimit is None or now > (lastevent["lastsent"] + self.__ratelimit))) + + if (self.__timeframe and (lastevent["lastseen"] > 0) + and (now >= (lastevent["lasteen"] + self.__timeframe))): + # If the time-frame have timed out, reset it + self.__lastseen = 0 + else: + self.__lastseen = now + else: + # Not seen before, register it as a new one + self.__matchedgroups[rghash] = { + "count": 1, + "lastseen": now, + "lastsent": 0 + } + ret = (1 % self.__threshold == 0) + + return ret + + + def ClearTimeTrackers(self, regexmatch): + rghash = self.__gen_hash(regexmatch) + if self.__matchedgroups.has_key(rghash): + self.__matchedgroups[rghash]["lasteen"] = 0 + self.__matchedgroups[rghash]["lastsent"] = 0 + + + def GetThreshold(self): + return self.__threshold + + + def GetCurrentCount(self, regexmatch): + rghash = self.__gen_hash(regexmatch) + if self.__matchedgroups.has_key(rghash): + return self.__matchedgroups[rghash]["count"] + else: + return 0 + + + +class ThresholdWatch(object): + WATCHTYPE_RULE = 1 + WATCHTYPE_EXACT = 2 + + def __init__(self, watchtype, params): + self.__watchtype = watchtype + if watchtype == self.WATCHTYPE_RULE: + self.__thresholdtype = ThresholdType_Rule(params) + elif watchtype == self.WATCHTYPE_EXACT: + self.__thresholdtype = ThresholdType_Exact(params) + else: + raise ValueError("Invalid watchtype parameter") + + + def CheckThreshold(self, alert, regexmatch): + return self.__thresholdtype.CheckThreshold(alert, regexmatch) + + + def ClearTimeTrackers(self, regexmatch): + self.__thresholdtype.ClearTimeTrackers(regexmatch) + + + def GetThreshold(self): + return self.__thresholdtype.GetThreshold() + + + def GetCurrentCount(self, regexmatch): + return self.__thresholdtype.GetCurrentCount(regexmatch) + diff --git a/LogActio/__init__.py b/LogActio/__init__.py index 5a7c167..a7f0c54 100644 --- a/LogActio/__init__.py +++ b/LogActio/__init__.py @@ -2,7 +2,7 @@ # logactio - simple framework for doing configured action on certain # log file events # -# Copyright 2012 David Sommerseth +# Copyright 2012 - 2013 David Sommerseth # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +26,7 @@ import sys, os, re, time, ConfigParser, threading, signal import ReporterQueue +from LogActio.ThresholdWatch import ThresholdWatch from LogActio.Reporters import DefaultReporter class WatcherThread(threading.Thread): @@ -43,18 +44,22 @@ class WatcherThread(threading.Thread): return self.__logfile - def AddRule(self, prefix, regex, threshold, timeframe, ratelimit, resetrules, reporters): + def AddRule(self, prefix, regex, thrtype, threshold, timeframe, ratelimit, resetrules, reporters): + # Convert threshold type from string to known internal variables + if thrtype is None or thrtype.lower() == "rule": + thrtype = ThresholdWatch.WATCHTYPE_RULE + elif thrtype.lower() == "exact": + thrtype = ThresholdWatch.WATCHTYPE_EXACT + # Adds a rule specific for this log file rule = {"prefix": prefix, "regex": re.compile(regex), - "threshold": int(threshold), - "timeframe": timeframe and int(timeframe) or None, - "ratelimit": ratelimit and int(ratelimit) or None, + "threshold": ThresholdWatch(thrtype, + {"threshold": threshold, + "timeframe": timeframe, + "ratelimit": ratelimit}), "resetrules": resetrules, - "lastseen": 0, - "current_count": 0, "alerts_sent": 0, - "lastsent": 0, "reporters": reporters} self.__rules.append(rule) @@ -105,25 +110,18 @@ class WatcherThread(threading.Thread): fp.seek(where) continue - now = int(time.time()) resetlist = [] for alert in self.__rules: m = alert["regex"].match(line.splitlines()[0]) # If the received log line matches the regex if not self.__shutdown and m: - alert["current_count"] += 1 - + regexmatch = m.groups() # If the threshold has been reached and within the given time frame, # report the incident. Also, if we have an rate-limit, only send # a report it is 'rate-limit seconds' since last report. - if (alert["threshold"] == 0 - or ((alert["current_count"] % alert["threshold"] == 0) - and (alert["timeframe"] is None - or now <= (alert["lastseen"] + alert["timeframe"]))) - and (alert["ratelimit"] is None or now > (alert["lastsent"] + alert["ratelimit"]))): + if alert["threshold"].CheckThreshold(alert, regexmatch): alert["alerts_sent"] += 1 - alert["lastsent"] = now - info = "|".join(m.groups()) # Gather regex exctracted info + info = "|".join(regexmatch) # Gather regex exctracted info if len(info) == 0: info = None @@ -132,31 +130,22 @@ class WatcherThread(threading.Thread): rep = alert.has_key("reporters") and alert["reporters"] or self.__reporters for r in rep: r.ProcessEvent(self.__logfile, alert["prefix"], info, - alert["current_count"], alert["threshold"]) + alert["threshold"].GetCurrentCount(regexmatch), alert["threshold"].GetThreshold()) # If reset-rule-rate-limits is set, make a note to reset these # counters after all alerts have been processed if alert["resetrules"]: for r in alert["resetrules"]: - resetlist.append(r) + resetlist.append((r, regexmatch)) - alert["lastseen"] = 0 continue - if (alert["timeframe"] and (alert["lastseen"] > 0) - and (now >= (alert["lastseen"] + alert["timeframe"]))): - # If the time-frame have timed out, reset it - alert["lastseen"] = 0 - else: - alert["lastseen"] = now - # If we have some reset tasks scheduled, perform them now for reset in resetlist: - for rule in self.__rules: + for (rule, rgmatch) in self.__rules: # Reset the lastsent and lastseen flags for the given rules if rule["prefix"] == reset: - rule["lastsent"] = 0 - rule["lastseen"] = 0 + rule["threshold"].ClearTimeTrackers(rgmatch) fp.close() return 0 @@ -349,6 +338,8 @@ class LogActio(object): # Add the rule to the proper WatchThread self.__watchthreads[idx].AddRule(rulename, self.__cfg.get(entry, "regex"), + (self.__cfg.has_option(entry, "threshold-type") + and self.__cfg.get(entry, "threshold-type") or None), self.__cfg.get(entry, "threshold"), (self.__cfg.has_option(entry, "time-frame") and self.__cfg.get(entry, "time-frame") or None), -- cgit