New upstream version 1.7.0+dfsgupstream/1.7.0+dfsg

1 files changed, 173 insertions, 94 deletions

diff --git a/python.d/fail2ban.chart.py b/python.d/fail2ban.chart.py
index c7d24e8c1..35761e894 100644
--- a/python.d/fail2ban.chart.py
+++ b/python.d/fail2ban.chart.py
@@ -2,131 +2,210 @@
 # Description: fail2ban log netdata python.d module
 # Author: l2isbad
 
-from base import LogService
-from re import compile
-
-try:
-    from itertools import filterfalse
-except ImportError:
-    from itertools import ifilterfalse as filterfalse
+from re import compile as r_compile
 from os import access as is_accessible, R_OK
-from os.path import isdir
+from os.path import isdir, getsize
 from glob import glob
+import bisect
+from base import LogService
 
 priority = 60000
 retries = 60
-REGEX = compile(r'\[([A-Za-z-_]+)][^\[\]]*?(?<!# )enabled = true')
-ORDER = ['jails_group']
+REGEX_JAILS = r_compile(r'\[([A-Za-z-_0-9]+)][^\[\]]*?(?<!# )enabled = (?:(true|false))')
+REGEX_DATA = r_compile(r'\[(?P<jail>[A-Za-z-_0-9]+)\] (?P<action>(?:(U|B)))[a-z]+ (?P<ipaddr>\d{1,3}(?:\.\d{1,3}){3})')
+ORDER = ['jails_bans', 'jails_in_jail']
 
 
 class Service(LogService):
+    """
+    fail2ban log class
+    Reads logs line by line
+    Jail auto detection included
+    It produces following charts:
+    * Bans per second for every jail
+    * Banned IPs for every jail (since the last restart of netdata)
+    """
     def __init__(self, configuration=None, name=None):
         LogService.__init__(self, configuration=configuration, name=name)
         self.order = ORDER
+        self.definitions = dict()
         self.log_path = self.configuration.get('log_path', '/var/log/fail2ban.log')
         self.conf_path = self.configuration.get('conf_path', '/etc/fail2ban/jail.local')
-        self.conf_dir = self.configuration.get('conf_dir', '')
-        try:
-            self.exclude = self.configuration['exclude'].split()
-        except (KeyError, AttributeError):
-            self.exclude = []
+        self.conf_dir = self.configuration.get('conf_dir', '/etc/fail2ban/jail.d/')
+        self.exclude = self.configuration.get('exclude')
 
     def _get_data(self):
         """
         Parse new log lines
         :return: dict
         """
-        try:
-            raw = self._get_raw_data()
-            if raw is None:
-                return None
-            elif not raw:
-                return self.data
-        except (ValueError, AttributeError):
+        raw = self._get_raw_data()
+        if raw is None:
             return None
+        elif not raw:
+            return self.to_netdata
 
         # Fail2ban logs looks like
         # 2016-12-25 12:36:04,711 fail2ban.actions[2455]: WARNING [ssh] Ban 178.156.32.231
-        data = dict(
-            zip(
-                self.jails_list,
-                [len(list(filterfalse(lambda line: (jail + '] Ban') not in line, raw))) for jail in self.jails_list]
-            ))
-
-        for jail in data:
-            self.data[jail] += data[jail]
-
-        return self.data
+        for row in raw:
+            match = REGEX_DATA.search(row)
+            if match:
+                match_dict = match.groupdict()
+                jail, action, ipaddr = match_dict['jail'], match_dict['action'], match_dict['ipaddr']
+                if jail in self.jails_list:
+                    if action == 'B':
+                        self.to_netdata[jail] += 1
+                        if address_not_in_jail(self.banned_ips[jail], ipaddr, self.to_netdata[jail + '_in_jail']):
+                            self.to_netdata[jail + '_in_jail'] += 1
+                    else:
+                        if ipaddr in self.banned_ips[jail]:
+                            self.banned_ips[jail].remove(ipaddr)
+                            self.to_netdata[jail + '_in_jail'] -= 1
+
+        return self.to_netdata
 
     def check(self):
+        """
+        :return: bool
 
-        # Check "log_path" is accessible.
-        # If NOT STOP plugin
-        if not is_accessible(self.log_path, R_OK):
-            self.error('Cannot access file %s' % self.log_path)
-            return False
-        jails_list = list()
-
-        if self.conf_dir:
-            dir_jails, error = parse_conf_dir(self.conf_dir)
-            jails_list.extend(dir_jails)
-            if not dir_jails:
-                self.error(error)
-
-        if self.conf_path:
-            path_jails, error = parse_conf_path(self.conf_path)
-            jails_list.extend(path_jails)
-            if not path_jails:
-                self.error(error)
+        Check if the "log_path" is not empty and readable
+        """
 
-        # If for some reason parse failed we still can START with default jails_list.
-        self.jails_list = list(set(jails_list) - set(self.exclude)) or ['ssh']
-        self.data = dict([(jail, 0) for jail in self.jails_list])
-        self.create_dimensions()
-        self.info('Plugin successfully started. Jails: %s' % self.jails_list)
+        if not (is_accessible(self.log_path, R_OK) and getsize(self.log_path) != 0):
+            self.error('%s is not readable or empty' % self.log_path)
+            return False
+        self.jails_list, self.to_netdata, self.banned_ips = self.jails_auto_detection_()
+        self.definitions = create_definitions_(self.jails_list)
+        self.info('Jails: %s' % self.jails_list)
         return True
 
-    def create_dimensions(self):
-        self.definitions = {
-            'jails_group': {'options': [None, "Jails ban statistics", "bans/s", 'jails', 'jail.ban', 'line'],
-                            'lines': []}}
-        for jail in self.jails_list:
-            self.definitions['jails_group']['lines'].append([jail, jail, 'incremental'])
-
+    def jails_auto_detection_(self):
+        """
+        return: <tuple>
 
-def parse_conf_dir(conf_dir):
-    if not isdir(conf_dir):
-        return list(), '%s is not a directory' % conf_dir
+        * jails_list - list of enabled jails (['ssh', 'apache', ...])
+        * to_netdata - dict ({'ssh': 0, 'ssh_in_jail': 0, ...})
+        * banned_ips - here will be stored all the banned ips ({'ssh': ['1.2.3.4', '5.6.7.8', ...], ...})
+        """
+        raw_jails_list = list()
+        jails_list = list()
 
-    jail_local = list(filter(lambda local: is_accessible(local, R_OK), glob(conf_dir + '/*.local')))
-    jail_conf = list(filter(lambda conf: is_accessible(conf, R_OK), glob(conf_dir + '/*.conf')))
+        for raw_jail in parse_configuration_files_(self.conf_path, self.conf_dir, self.error):
+            raw_jails_list.extend(raw_jail)
 
-    if not (jail_local or jail_conf):
-        return list(), '%s is empty or not readable' % conf_dir
+        for jail, status in raw_jails_list:
+            if status == 'true' and jail not in jails_list:
+                jails_list.append(jail)
+            elif status == 'false' and jail in jails_list:
+                jails_list.remove(jail)
 
-    # According "man jail.conf" files could be *.local AND *.conf
-    # *.conf files parsed first. Changes in *.local overrides configuration in *.conf
-    if jail_conf:
-        jail_local.extend([conf for conf in jail_conf if conf[:-5] not in [local[:-6] for local in jail_local]])
+        # If for some reason parse failed we still can START with default jails_list.
+        jails_list = list(set(jails_list) - set(self.exclude.split()
+                                                if isinstance(self.exclude, str) else list())) or ['ssh']
+
+        to_netdata = dict([(jail, 0) for jail in jails_list])
+        to_netdata.update(dict([(jail + '_in_jail', 0) for jail in jails_list]))
+        banned_ips = dict([(jail, list()) for jail in jails_list])
+
+        return jails_list, to_netdata, banned_ips
+
+
+def create_definitions_(jails_list):
+    """
+    Chart definitions creating
+    """
+
+    definitions = {
+        'jails_bans': {'options': [None, 'Jails Ban Statistics', 'bans/s', 'bans', 'jail.bans', 'line'],
+                       'lines': []},
+        'jails_in_jail': {'options': [None, 'Banned IPs (since the last restart of netdata)', 'IPs',
+                                      'in jail', 'jail.in_jail', 'line'],
+                          'lines': []}}
+    for jail in jails_list:
+        definitions['jails_bans']['lines'].append([jail, jail, 'incremental'])
+        definitions['jails_in_jail']['lines'].append([jail + '_in_jail', jail, 'absolute'])
+
+    return definitions
+
+
+def parse_configuration_files_(jails_conf_path, jails_conf_dir, print_error):
+    """
+    :param jails_conf_path: <str>
+    :param jails_conf_dir: <str>
+    :param print_error: <function>
+    :return: <tuple>
+
+    Uses "find_jails_in_files" function to find all jails in the "jails_conf_dir" directory
+    and in the "jails_conf_path"
+
+    All files must endswith ".local" or ".conf"
+    Return order is important.
+    According man jail.conf it should be
+    * jail.conf
+    * jail.d/*.conf (in alphabetical order)
+    * jail.local
+    * jail.d/*.local (in alphabetical order)
+    """
+    path_conf, path_local, dir_conf, dir_local = list(), list(), list(), list()
+
+    # Parse files in the directory
+    if not (isinstance(jails_conf_dir, str) and isdir(jails_conf_dir)):
+        print_error('%s is not a directory' % jails_conf_dir)
+    else:
+        dir_conf = list(filter(lambda conf: is_accessible(conf, R_OK), glob(jails_conf_dir + '/*.conf')))
+        dir_local = list(filter(lambda local: is_accessible(local, R_OK), glob(jails_conf_dir + '/*.local')))
+        if not (dir_conf or dir_local):
+            print_error('%s is empty or not readable' % jails_conf_dir)
+        else:
+            dir_conf, dir_local = (find_jails_in_files(dir_conf, print_error),
+                                   find_jails_in_files(dir_local, print_error))
+
+    # Parse .conf and .local files
+    if isinstance(jails_conf_path, str) and jails_conf_path.endswith(('.local', '.conf')):
+        path_conf, path_local = (find_jails_in_files([jails_conf_path.split('.')[0] + '.conf'], print_error),
+                                 find_jails_in_files([jails_conf_path.split('.')[0] + '.local'], print_error))
+
+    return path_conf, dir_conf, path_local, dir_local
+
+
+def find_jails_in_files(list_of_files, print_error):
+    """
+    :param list_of_files: <list>
+    :param print_error: <function>
+    :return: <list>
+
+    Open a file and parse it to find all (enabled and disabled) jails
+    The output is a list of tuples:
+    [('ssh', 'true'), ('apache', 'false'), ...]
+    """
     jails_list = list()
-    for conf in jail_local:
-        with open(conf, 'rt') as f:
-            raw_data = f.read()
-
-        data = ' '.join(raw_data.split())
-        jails_list.extend(REGEX.findall(data))
-    jails_list = list(set(jails_list))
-
-    return jails_list, 'can\'t locate any jails in %s. Default jail is [\'ssh\']' % conf_dir
-
-
-def parse_conf_path(conf_path):
-    if not is_accessible(conf_path, R_OK):
-        return list(), '%s is not readable' % conf_path
-
-    with open(conf_path, 'rt') as jails_conf:
-        raw_data = jails_conf.read()
-
-    data = raw_data.split()
-    jails_list = REGEX.findall(' '.join(data))
-    return jails_list, 'can\'t locate any jails in %s. Default jail is  [\'ssh\']' % conf_path
+    for conf in list_of_files:
+        if is_accessible(conf, R_OK):
+            with open(conf, 'rt') as conf:
+                raw_data = conf.read()
+            data = ' '.join(raw_data.split())
+            jails_list.extend(REGEX_JAILS.findall(data))
+        else:
+            print_error('%s is not readable or not exist' % conf)
+    return jails_list
+
+
+def address_not_in_jail(pool, address, pool_size):
+    """
+    :param pool: <list>
+    :param address: <str>
+    :param pool_size: <int>
+    :return: bool
+
+    Checks if the address is in the pool.
+    If not address will be added
+    """
+    index = bisect.bisect_left(pool, address)
+    if index < pool_size:
+        if pool[index] == address:
+            return False
+        bisect.insort_left(pool, address)
+        return True
+    else:
+        bisect.insort_left(pool, address)
+        return True