Merging upstream version 1.36.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

20 files changed, 164 insertions, 656 deletions

diff --git a/collectors/python.d.plugin/Makefile.am b/collectors/python.d.plugin/Makefile.am
index 667f1627c..9377ebe8d 100644
--- a/collectors/python.d.plugin/Makefile.am
+++ b/collectors/python.d.plugin/Makefile.am
@@ -48,7 +48,6 @@ include bind_rndc/Makefile.inc
 include boinc/Makefile.inc
 include ceph/Makefile.inc
 include changefinder/Makefile.inc
-include chrony/Makefile.inc
 include dockerd/Makefile.inc
 include dovecot/Makefile.inc
 include example/Makefile.inc
@@ -71,7 +70,6 @@ include nginx_plus/Makefile.inc
 include nvidia_smi/Makefile.inc
 include nsd/Makefile.inc
 include ntpd/Makefile.inc
-include ovpn_status_log/Makefile.inc
 include openldap/Makefile.inc
 include oracledb/Makefile.inc
 include postfix/Makefile.inc
diff --git a/collectors/python.d.plugin/chrony/Makefile.inc b/collectors/python.d.plugin/chrony/Makefile.inc
deleted file mode 100644
index 18a805b12..000000000
--- a/collectors/python.d.plugin/chrony/Makefile.inc
+++ /dev/null
@@ -1,13 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# THIS IS NOT A COMPLETE Makefile
-# IT IS INCLUDED BY ITS PARENT'S Makefile.am
-# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
-
-# install these files
-dist_python_DATA       += chrony/chrony.chart.py
-dist_pythonconfig_DATA += chrony/chrony.conf
-
-# do not install these files, but include them in the distribution
-dist_noinst_DATA       += chrony/README.md chrony/Makefile.inc
-
diff --git a/collectors/python.d.plugin/chrony/README.md b/collectors/python.d.plugin/chrony/README.md
deleted file mode 100644
index 3093ec3f9..000000000
--- a/collectors/python.d.plugin/chrony/README.md
+++ /dev/null
@@ -1,61 +0,0 @@
-<!--
-title: "Chrony monitoring with Netdata"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/chrony/README.md
-sidebar_label: "Chrony"
--->
-
-# Chrony monitoring with Netdata
-
-Monitors the precision and statistics of a local chronyd server, and produces:
-
--   frequency
--   last offset
--   RMS offset
--   residual freq
--   root delay
--   root dispersion
--   skew
--   system time
-
-## Requirements
-
-Verify that user Netdata can execute `chronyc tracking`. If necessary, update `/etc/chrony.conf`, `cmdallow`.
-
-## Enable the collector
-
-The `chrony` collector is disabled by default. To enable it, use `edit-config` from the Netdata [config
-directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf` file.
-
-```bash
-cd /etc/netdata   # Replace this path with your Netdata config directory, if different
-sudo ./edit-config python.d.conf
-```
-
-Change the value of the `chrony` setting to `yes`. Save the file and restart the Netdata Agent with `sudo systemctl
-restart netdata`, or the appropriate method for your system, to finish enabling the `chrony` collector.
-
-## Configuration
-
-Edit the `python.d/chrony.conf` configuration file using `edit-config` from the Netdata [config
-directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
-
-```bash
-cd /etc/netdata   # Replace this path with your Netdata config directory, if different
-sudo ./edit-config python.d/chrony.conf
-```
-
-Sample:
-
-```yaml
-# data collection frequency:
-update_every: 1
-
-# chrony query command:
-local:
-  command: 'chronyc -n tracking'
-```
-
-Save the file and restart the Netdata Agent with `sudo systemctl restart netdata`, or the [appropriate
-method](/docs/configure/start-stop-restart.md) for your system, to finish configuring the `chrony` collector.
-
-
diff --git a/collectors/python.d.plugin/chrony/chrony.chart.py b/collectors/python.d.plugin/chrony/chrony.chart.py
deleted file mode 100644
index 91f725001..000000000
--- a/collectors/python.d.plugin/chrony/chrony.chart.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# -*- coding: utf-8 -*-
-# Description: chrony netdata python.d module
-# Author: Dominik Schloesser (domschl)
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-from bases.FrameworkServices.ExecutableService import ExecutableService
-
-# default module values (can be overridden per job in `config`)
-update_every = 5
-
-CHRONY_COMMAND = 'chronyc -n tracking'
-
-# charts order (can be overridden if you want less charts, or different order)
-ORDER = [
-    'system',
-    'offsets',
-    'stratum',
-    'root',
-    'frequency',
-    'residualfreq',
-    'skew',
-]
-
-CHARTS = {
-    'system': {
-        'options': [None, 'Chrony System Time Deltas', 'microseconds', 'system', 'chrony.system', 'area'],
-        'lines': [
-            ['timediff', 'system time', 'absolute', 1, 1000]
-        ]
-    },
-    'offsets': {
-        'options': [None, 'Chrony System Time Offsets', 'microseconds', 'system', 'chrony.offsets', 'area'],
-        'lines': [
-            ['lastoffset', 'last offset', 'absolute', 1, 1000],
-            ['rmsoffset', 'RMS offset', 'absolute', 1, 1000]
-        ]
-    },
-    'stratum': {
-        'options': [None, 'Chrony Stratum', 'stratum', 'root', 'chrony.stratum', 'line'],
-        'lines': [
-            ['stratum', None, 'absolute', 1, 1]
-        ]
-    },
-    'root': {
-        'options': [None, 'Chrony Root Delays', 'milliseconds', 'root', 'chrony.root', 'line'],
-        'lines': [
-            ['rootdelay', 'delay', 'absolute', 1, 1000000],
-            ['rootdispersion', 'dispersion', 'absolute', 1, 1000000]
-        ]
-    },
-    'frequency': {
-        'options': [None, 'Chrony Frequency', 'ppm', 'frequencies', 'chrony.frequency', 'area'],
-        'lines': [
-            ['frequency', None, 'absolute', 1, 1000]
-        ]
-    },
-    'residualfreq': {
-        'options': [None, 'Chrony Residual frequency', 'ppm', 'frequencies', 'chrony.residualfreq', 'area'],
-        'lines': [
-            ['residualfreq', 'residual frequency', 'absolute', 1, 1000]
-        ]
-    },
-    'skew': {
-        'options': [None, 'Chrony Skew, error bound on frequency', 'ppm', 'frequencies', 'chrony.skew', 'area'],
-        'lines': [
-            ['skew', None, 'absolute', 1, 1000]
-        ]
-    }
-}
-
-CHRONY = [
-    ('Frequency', 'frequency', 1e3),
-    ('Last offset', 'lastoffset', 1e9),
-    ('RMS offset', 'rmsoffset', 1e9),
-    ('Residual freq', 'residualfreq', 1e3),
-    ('Root delay', 'rootdelay', 1e9),
-    ('Root dispersion', 'rootdispersion', 1e9),
-    ('Skew', 'skew', 1e3),
-    ('Stratum', 'stratum', 1),
-    ('System time', 'timediff', 1e9)
-]
-
-
-class Service(ExecutableService):
-    def __init__(self, configuration=None, name=None):
-        ExecutableService.__init__(
-            self, configuration=configuration, name=name)
-        self.order = ORDER
-        self.definitions = CHARTS
-        self.command = CHRONY_COMMAND
-
-    def _get_data(self):
-        """
-        Format data received from shell command
-        :return: dict
-        """
-        raw_data = self._get_raw_data()
-        if not raw_data:
-            return None
-
-        raw_data = (line.split(':', 1) for line in raw_data)
-        parsed, data = dict(), dict()
-
-        for line in raw_data:
-            try:
-                key, value = (l.strip() for l in line)
-            except ValueError:
-                continue
-            if value:
-                parsed[key] = value.split()[0]
-
-        for key, dim_id, multiplier in CHRONY:
-            try:
-                data[dim_id] = int(float(parsed[key]) * multiplier)
-            except (KeyError, ValueError):
-                continue
-
-        return data or None
diff --git a/collectors/python.d.plugin/chrony/chrony.conf b/collectors/python.d.plugin/chrony/chrony.conf
deleted file mode 100644
index fd95519b5..000000000
--- a/collectors/python.d.plugin/chrony/chrony.conf
+++ /dev/null
@@ -1,77 +0,0 @@
-# netdata python.d.plugin configuration for chrony 
-#
-# This file is in YaML format. Generally the format is:
-#
-# name: value
-#
-# There are 2 sections:
-#  - global variables
-#  - one or more JOBS
-#
-# JOBS allow you to collect values from multiple sources.
-# Each source will have its own set of charts.
-#
-# JOB parameters have to be indented (using spaces only, example below).
-
-# ----------------------------------------------------------------------
-# Global Variables
-# These variables set the defaults for all JOBs, however each JOB
-# may define its own, overriding the defaults.
-
-# update_every sets the default data collection frequency.
-# If unset, the python.d.plugin default is used.
-update_every: 5
-
-# priority controls the order of charts at the netdata dashboard.
-# Lower numbers move the charts towards the top of the page.
-# If unset, the default for python.d.plugin is used.
-# priority: 60000
-
-# penalty indicates whether to apply penalty to update_every in case of failures.
-# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
-# penalty: yes
-
-# autodetection_retry sets the job re-check interval in seconds.
-# The job is not deleted if check fails.
-# Attempts to start the job are made once every autodetection_retry.
-# This feature is disabled by default.
-# autodetection_retry: 0
-
-# ----------------------------------------------------------------------
-# JOBS (data collection sources)
-#
-# The default JOBS share the same *name*. JOBS with the same name
-# are mutually exclusive. Only one of them will be allowed running at
-# any time. This allows autodetection to try several alternatives and
-# pick the one that works.
-#
-# Any number of jobs is supported.
-#
-# All python.d.plugin JOBS (for all its modules) support a set of
-# predefined parameters. These are:
-#
-# job_name:
-#     name: myname            # the JOB's name as it will appear at the
-#                             # dashboard (by default is the job_name)
-#                             # JOBs sharing a name are mutually exclusive
-#     update_every: 1         # the JOB's data collection frequency
-#     priority: 60000         # the JOB's order on the dashboard
-#     penalty: yes            # the JOB's penalty
-#     autodetection_retry: 0  # the JOB's re-check interval in seconds
-#
-# Additionally to the above, chrony also supports the following:
-#
-#     command: 'chrony tracking' # the command to run
-#
-
-# ----------------------------------------------------------------------
-# REQUIRED chrony CONFIGURATION
-#
-# netdata will query chrony as user netdata.
-# verify that user netdata is allowed to call 'chronyc tracking'
-# Check cmdallow in chrony.conf
-# ----------------------------------------------------------------------
-# AUTO-DETECTION JOBS
-
-local:
-  command: 'chronyc -n tracking'
diff --git a/collectors/python.d.plugin/haproxy/README.md b/collectors/python.d.plugin/haproxy/README.md
index 4eb962e24..f16e7258e 100644
--- a/collectors/python.d.plugin/haproxy/README.md
+++ b/collectors/python.d.plugin/haproxy/README.md
@@ -9,30 +9,32 @@ sidebar_label: "HAProxy"
 Monitors frontend and backend metrics such as bytes in, bytes out, sessions current, sessions in queue current.
 And health metrics such as backend servers status (server check should be used).
 
-Plugin can obtain data from url **OR** unix socket.
+Plugin can obtain data from URL or Unix socket.
 
-**Requirement:**
-Socket MUST be readable AND writable by the `netdata` user.
+Requirement:
+
+- Socket must be readable and writable by the `netdata` user.
+- URL must have `stats uri <path>` present in the haproxy config, otherwise you will get HTTP 503 in the haproxy logs.
 
 It produces:
 
-1.  **Frontend** family charts
+1. **Frontend** family charts
 
-    -   Kilobytes in/s
-    -   Kilobytes out/s
-    -   Sessions current
-    -   Sessions in queue current
+    - Kilobytes in/s
+    - Kilobytes out/s
+    - Sessions current
+    - Sessions in queue current
 
-2.  **Backend** family charts
+2. **Backend** family charts
 
-    -   Kilobytes in/s
-    -   Kilobytes out/s
-    -   Sessions current
-    -   Sessions in queue current
+    - Kilobytes in/s
+    - Kilobytes out/s
+    - Sessions current
+    - Sessions in queue current
 
-3.  **Health** chart
+3. **Health** chart
 
-    -   number of failed servers for every backend (in DOWN state)
+    - number of failed servers for every backend (in DOWN state)
 
 ## Configuration
 
@@ -48,20 +50,18 @@ Sample:
 
 ```yaml
 via_url:
-  user       : 'username' # ONLY IF stats auth is used
-  pass       : 'password' # # ONLY IF stats auth is used
-  url     : 'http://ip.address:port/url;csv;norefresh'
+  user: 'username' # ONLY IF stats auth is used
+  pass: 'password' # # ONLY IF stats auth is used
+  url: 'http://ip.address:port/url;csv;norefresh'
 ```
 
 OR
 
 ```yaml
 via_socket:
-  socket       : 'path/to/haproxy/sock'
+  socket: 'path/to/haproxy/sock'
 ```
 
 If no configuration is given, module will fail to run.
 
 ---
-
-
diff --git a/collectors/python.d.plugin/mongodb/mongodb.chart.py b/collectors/python.d.plugin/mongodb/mongodb.chart.py
index bec94d3ef..5e8fec834 100644
--- a/collectors/python.d.plugin/mongodb/mongodb.chart.py
+++ b/collectors/python.d.plugin/mongodb/mongodb.chart.py
@@ -10,7 +10,7 @@ from datetime import datetime
 from sys import exc_info
 
 try:
-    from pymongo import MongoClient, ASCENDING, DESCENDING
+    from pymongo import MongoClient, ASCENDING, DESCENDING, version_tuple
     from pymongo.errors import PyMongoError
 
     PYMONGO = True
@@ -750,7 +750,7 @@ class Service(SimpleService):
             CONN_PARAM_HOST: conf.get(CONN_PARAM_HOST, DEFAULT_HOST),
             CONN_PARAM_PORT: conf.get(CONN_PARAM_PORT, DEFAULT_PORT),
         }
-        if hasattr(MongoClient, 'server_selection_timeout'):
+        if hasattr(MongoClient, 'server_selection_timeout') or version_tuple[0] >= 4:
             params[CONN_PARAM_SERVER_SELECTION_TIMEOUT_MS] = conf.get('timeout', DEFAULT_TIMEOUT)
 
         params.update(self.build_ssl_connection_params())
diff --git a/collectors/python.d.plugin/ovpn_status_log/Makefile.inc b/collectors/python.d.plugin/ovpn_status_log/Makefile.inc
deleted file mode 100644
index 1fbc506d6..000000000
--- a/collectors/python.d.plugin/ovpn_status_log/Makefile.inc
+++ /dev/null
@@ -1,13 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# THIS IS NOT A COMPLETE Makefile
-# IT IS INCLUDED BY ITS PARENT'S Makefile.am
-# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
-
-# install these files
-dist_python_DATA       += ovpn_status_log/ovpn_status_log.chart.py
-dist_pythonconfig_DATA += ovpn_status_log/ovpn_status_log.conf
-
-# do not install these files, but include them in the distribution
-dist_noinst_DATA       += ovpn_status_log/README.md ovpn_status_log/Makefile.inc
-
diff --git a/collectors/python.d.plugin/ovpn_status_log/README.md b/collectors/python.d.plugin/ovpn_status_log/README.md
deleted file mode 100644
index 374d1910e..000000000
--- a/collectors/python.d.plugin/ovpn_status_log/README.md
+++ /dev/null
@@ -1,50 +0,0 @@
-<!--
-title: "OpenVPN monitoring with Netdata"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/ovpn_status_log/README.md
-sidebar_label: "OpenVPN"
--->
-
-# OpenVPN monitoring with Netdata
-
-Parses server log files and provides summary (client, traffic) metrics.
-
-## Requirements
-
--   If you are running multiple OpenVPN instances out of the same directory, MAKE SURE TO EDIT DIRECTIVES which create output files
-    so that multiple instances do not overwrite each other's output files.
-
--   Make sure NETDATA USER CAN READ openvpn-status.log
-
--   Update_every interval MUST MATCH interval on which OpenVPN writes operational status to log file.
-
-It produces:
-
-1.  **Users** OpenVPN active users
-
-    -   users
-
-2.  **Traffic** OpenVPN overall bandwidth usage in kilobit/s
-
-    -   in
-    -   out
-
-## Configuration
-
-Edit the `python.d/ovpn_status_log.conf` configuration file using `edit-config` from the Netdata [config
-directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
-
-```bash
-cd /etc/netdata   # Replace this path with your Netdata config directory, if different
-sudo ./edit-config python.d/ovpn_status_log.conf
-```
-
-Sample:
-
-```yaml
-default
- log_path     : '/var/log/openvpn-status.log'
-```
-
----
-
-
diff --git a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.chart.py b/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.chart.py
deleted file mode 100644
index cfc87be36..000000000
--- a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.chart.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# -*- coding: utf-8 -*-
-# Description: openvpn status log netdata python.d module
-# Author: ilyam8
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import re
-
-from bases.FrameworkServices.SimpleService import SimpleService
-
-update_every = 10
-
-ORDER = [
-    'users',
-    'traffic',
-]
-
-CHARTS = {
-    'users': {
-        'options': [None, 'OpenVPN Active Users', 'active users', 'users', 'openvpn_status.users', 'line'],
-        'lines': [
-            ['users', None, 'absolute'],
-        ]
-    },
-    'traffic': {
-        'options': [None, 'OpenVPN Traffic', 'KiB/s', 'traffic', 'openvpn_status.traffic', 'area'],
-        'lines': [
-            ['bytes_in', 'in', 'incremental', 1, 1 << 10],
-            ['bytes_out', 'out', 'incremental', -1, 1 << 10]
-        ]
-    }
-}
-
-TLS_REGEX = re.compile(
-    r'(?:[0-9a-f]+:[0-9a-f:]+|(?:\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?)) (?P<bytes_in>\d+) (?P<bytes_out>\d+)'
-)
-STATIC_KEY_REGEX = re.compile(
-    r'TCP/[A-Z]+ (?P<direction>(?:read|write)) bytes,(?P<bytes>\d+)'
-)
-
-
-class Service(SimpleService):
-    def __init__(self, configuration=None, name=None):
-        SimpleService.__init__(self, configuration=configuration, name=name)
-        self.order = ORDER
-        self.definitions = CHARTS
-        self.log_path = self.configuration.get('log_path')
-        self.regex = {
-            'tls': TLS_REGEX,
-            'static_key': STATIC_KEY_REGEX
-        }
-
-    def check(self):
-        if not (self.log_path and isinstance(self.log_path, str)):
-            self.error("'log_path' is not defined")
-            return False
-
-        data = self._get_raw_data()
-        if not data:
-            self.error('Make sure that the openvpn status log file exists and netdata has permission to read it')
-            return None
-
-        found = None
-        for row in data:
-            if 'ROUTING' in row:
-                self.get_data = self.get_data_tls
-                found = True
-                break
-            elif 'STATISTICS' in row:
-                self.get_data = self.get_data_static_key
-                found = True
-                break
-        if found:
-            return True
-        self.error('Failed to parse openvpn log file')
-        return False
-
-    def _get_raw_data(self):
-        """
-        Open log file
-        :return: str
-        """
-
-        try:
-            with open(self.log_path) as log:
-                raw_data = log.readlines() or None
-        except OSError:
-            return None
-        else:
-            return raw_data
-
-    def get_data_static_key(self):
-        """
-        Parse openvpn-status log file.
-        """
-
-        raw_data = self._get_raw_data()
-        if not raw_data:
-            return None
-
-        data = dict(bytes_in=0, bytes_out=0)
-
-        for row in raw_data:
-            match = self.regex['static_key'].search(row)
-            if match:
-                match = match.groupdict()
-                if match['direction'] == 'read':
-                    data['bytes_in'] += int(match['bytes'])
-                else:
-                    data['bytes_out'] += int(match['bytes'])
-
-        return data or None
-
-    def get_data_tls(self):
-        """
-        Parse openvpn-status log file.
-        """
-
-        raw_data = self._get_raw_data()
-        if not raw_data:
-            return None
-
-        data = dict(users=0, bytes_in=0, bytes_out=0)
-        for row in raw_data:
-            columns = row.split(',') if ',' in row else row.split()
-            if 'UNDEF' in columns:
-                # see https://openvpn.net/archive/openvpn-users/2004-08/msg00116.html
-                continue
-
-            match = self.regex['tls'].search(' '.join(columns))
-            if match:
-                match = match.groupdict()
-                data['users'] += 1
-                data['bytes_in'] += int(match['bytes_in'])
-                data['bytes_out'] += int(match['bytes_out'])
-
-        return data or None
diff --git a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.conf b/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.conf
deleted file mode 100644
index 1d71f6b8e..000000000
--- a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.conf
+++ /dev/null
@@ -1,97 +0,0 @@
-# netdata python.d.plugin configuration for openvpn status log
-#
-# This file is in YaML format. Generally the format is:
-#
-# name: value
-#
-# There are 2 sections:
-#  - global variables
-#  - one or more JOBS
-#
-# JOBS allow you to collect values from multiple sources.
-# Each source will have its own set of charts.
-#
-# JOB parameters have to be indented (using spaces only, example below).
-
-# ----------------------------------------------------------------------
-# Global Variables
-# These variables set the defaults for all JOBs, however each JOB
-# may define its own, overriding the defaults.
-
-# update_every sets the default data collection frequency.
-# If unset, the python.d.plugin default is used.
-# update_every: 1
-
-# priority controls the order of charts at the netdata dashboard.
-# Lower numbers move the charts towards the top of the page.
-# If unset, the default for python.d.plugin is used.
-# priority: 60000
-
-# penalty indicates whether to apply penalty to update_every in case of failures.
-# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.
-# penalty: yes
-
-# autodetection_retry sets the job re-check interval in seconds.
-# The job is not deleted if check fails.
-# Attempts to start the job are made once every autodetection_retry.
-# This feature is disabled by default.
-# autodetection_retry: 0
-
-# ----------------------------------------------------------------------
-# JOBS (data collection sources)
-#
-# The default JOBS share the same *name*. JOBS with the same name
-# are mutually exclusive. Only one of them will be allowed running at
-# any time. This allows autodetection to try several alternatives and
-# pick the one that works.
-#
-# Any number of jobs is supported.
-#
-# All python.d.plugin JOBS (for all its modules) support a set of
-# predefined parameters. These are:
-#
-# job_name:
-#     name: myname            # the JOB's name as it will appear at the
-#                             # dashboard (by default is the job_name)
-#                             # JOBs sharing a name are mutually exclusive
-#     update_every: 1         # the JOB's data collection frequency
-#     priority: 60000         # the JOB's order on the dashboard
-#     penalty: yes            # the JOB's penalty
-#     autodetection_retry: 0  # the JOB's re-check interval in seconds
-#
-# Additionally to the above, openvpn status log also supports the following:
-#
-#     log_path: 'PATH'     # the path to openvpn status log file
-#
-# ----------------------------------------------------------------------
-# AUTO-DETECTION JOBS
-# only one of them will run (they have the same name)
-#
-# IMPORTANT information
-#
-# 1. If you are running multiple OpenVPN instances out of the same directory, MAKE SURE TO EDIT DIRECTIVES which create output files
-# so that multiple instances do not overwrite each other's output files.
-# 2. Make sure NETDATA USER CAN READ openvpn-status.log
-#
-#     * cd into directory with openvpn-status.log and run the following commands as root 
-#     * #chown :netdata openvpn-status.log && chmod 640 openvpn-status.log
-#     * To check permission and group membership run
-#     * #ls -l openvpn-status.log
-#     -rw-r----- 1 root netdata 359 dec 21 21:22 openvpn-status.log
-#
-# 3. Update_every interval MUST MATCH interval on which OpenVPN writes operational status to log file.
-# If its not true traffic chart WILL DISPLAY WRONG values
-#
-# Default OpenVPN update interval is 10 second on Debian 8
-# # ps -C openvpn -o command=
-#       /usr/sbin/openvpn --daemon ovpn-server --status /run/openvpn/server.status 10 --cd /etc/openvpn --config /etc/openvpn/server.conf
-# 
-# 4. Confirm status is configured in your OpenVPN configuration.
-#     * Open OpenVPN config in an editor (e.g. sudo nano /etc/openvpn/default.conf)
-#     * Confirm status is enabled with below:
-#     status /var/log/openvpn-status.log
-#
-#default:
-# log_path: '/var/log/openvpn-status.log'
-#
-# ----------------------------------------------------------------------
diff --git a/collectors/python.d.plugin/postgres/README.md b/collectors/python.d.plugin/postgres/README.md
index 224b76ff5..7acb9a7a9 100644
--- a/collectors/python.d.plugin/postgres/README.md
+++ b/collectors/python.d.plugin/postgres/README.md
@@ -6,6 +6,9 @@ sidebar_label: "PostgreSQL"
 
 # PostgreSQL monitoring with Netdata
 
+> **Warning**: This module is deprecated and will be deleted in v1.37.0.
+> Use [go.d/postgres](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/postgres).
+
 Collects database health and performance metrics.
 
 ## Requirements
@@ -97,7 +100,8 @@ cd /etc/netdata   # Replace this path with your Netdata config directory, if dif
 sudo ./edit-config python.d/postgres.conf
 ```
 
-When no configuration file is found, the module tries to connect to TCP/IP socket: `localhost:5432`.
+When no configuration file is found, the module tries to connect to TCP/IP socket: `localhost:5432` with the 
+following collection jobs.
 
 ```yaml
 socket:
@@ -113,6 +117,29 @@ tcp:
   port         : 5432
 ```
 
+**Note**: Every job collection must have a unique identifier. In cases that you monitor multiple DBs, every
+job must have it's own name. Use a mnemonic of your preference (e.g us_east_db, us_east_tcp)
+
+## Troubleshooting
+
+To troubleshoot issues with the `postgres` collector, run the `python.d.plugin` with the debug option enabled. The output
+should give you clues as to why the collector isn't working.
+
+First, navigate to your plugins directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on your
+system, open `netdata.conf` and look for the setting `plugins directory`. Once you're in the plugin's directory, switch
+to the `netdata` user.
+
+```bash
+cd /usr/libexec/netdata/plugins.d/
+sudo su -s /bin/bash netdata
+```
+
+You can now run the `python.d.plugin` to debug the collector:
+
+```bash
+./python.d.plugin postgres debug trace
+```
+
 ---
 
 
diff --git a/collectors/python.d.plugin/python.d.conf b/collectors/python.d.plugin/python.d.conf
index af58b451c..72e20fcd3 100644
--- a/collectors/python.d.plugin/python.d.conf
+++ b/collectors/python.d.plugin/python.d.conf
@@ -25,73 +25,48 @@ gc_run: yes
 # Garbage collection interval in seconds. Default is 300.
 gc_interval: 300
 
-# apache: yes
-
-# apache_cache has been replaced by web_log
 # adaptec_raid: yes
 # alarms: yes
 # am2320: yes
 # anomalies: no
-apache_cache: no
 # beanstalk: yes
 # bind_rndc: yes
 # boinc: yes
 # ceph: yes
-chrony: no
 # changefinder: no
-# couchdb: yes
-# dns_query_time: yes
-# dnsdist: yes
 # dockerd: yes
 # dovecot: yes
-# elasticsearch: yes
-# energid: yes
 
 # this is just an example
 example: no
 
 # exim: yes
 # fail2ban: yes
-# freeradius: yes
 # gearman: yes
 go_expvar: no
 
-# gunicorn_log has been replaced by web_log
-gunicorn_log: no
 # haproxy: yes
 # hddtemp: yes
-# httpcheck: yes
 hpssa: no
 # icecast: yes
 # ipfs: yes
-# isc_dhcpd: yes
 # litespeed: yes
 logind: no
 # megacli: yes
 # memcached: yes
 # mongodb: yes
 # monit: yes
-# mysql: yes
-# nginx: yes
 # nginx_plus: yes
 # nvidia_smi: yes
-
-# nginx_log has been replaced by web_log
-nginx_log: no
 # nsd: yes
 # ntpd: yes
 # openldap: yes
 # oracledb: yes
-# ovpn_status_log: yes
-# phpfpm: yes
-# portcheck: yes
 # postfix: yes
 # postgres: yes
-# powerdns: yes
 # proxysql: yes
 # puppet: yes
 # rabbitmq: yes
-# redis: yes
 # rethinkdbs: yes
 # retroshare: yes
 # riakkv: yes
@@ -107,5 +82,4 @@ nginx_log: no
 # uwsgi: yes
 # varnish: yes
 # w1sensor: yes
-# web_log: yes
 # zscores: no
diff --git a/collectors/python.d.plugin/python.d.plugin.in b/collectors/python.d.plugin/python.d.plugin.in
index b943f3a20..c04cb3ff0 100644
--- a/collectors/python.d.plugin/python.d.plugin.in
+++ b/collectors/python.d.plugin/python.d.plugin.in
@@ -31,8 +31,8 @@ import os
 import pprint
 import re
 import sys
-import time
 import threading
+import time
 import types
 
 try:
@@ -50,6 +50,7 @@ else:
 ENV_NETDATA_USER_CONFIG_DIR = 'NETDATA_USER_CONFIG_DIR'
 ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR'
 ENV_NETDATA_PLUGINS_DIR = 'NETDATA_PLUGINS_DIR'
+ENV_NETDATA_USER_PLUGINS_DIRS = 'NETDATA_USER_PLUGINS_DIRS'
 ENV_NETDATA_LIB_DIR = 'NETDATA_LIB_DIR'
 ENV_NETDATA_UPDATE_EVERY = 'NETDATA_UPDATE_EVERY'
 ENV_NETDATA_LOCK_DIR = 'NETDATA_LOCK_DIR'
@@ -99,6 +100,9 @@ def dirs():
     modules_user_config = os.path.join(plugin_user_config, 'python.d')
     modules_stock_config = os.path.join(plugin_stock_config, 'python.d')
     modules = os.path.abspath(pluginsd + '/../python.d')
+    user_modules = [os.path.join(p, 'python.d') for p in
+                    os.getenv(ENV_NETDATA_USER_PLUGINS_DIRS, "").split(" ") if
+                    p]
 
     Dirs = collections.namedtuple(
         'Dirs',
@@ -108,6 +112,7 @@ def dirs():
             'modules_user_config',
             'modules_stock_config',
             'modules',
+            'user_modules',
             'var_lib',
             'locks',
         ]
@@ -118,6 +123,7 @@ def dirs():
         modules_user_config,
         modules_stock_config,
         modules,
+        user_modules,
         var_lib,
         locks,
     )
@@ -130,6 +136,28 @@ IS_ATTY = sys.stdout.isatty()
 MODULE_SUFFIX = '.chart.py'
 
 
+def find_available_modules(*directories):
+    AvailableModule = collections.namedtuple(
+        'AvailableModule',
+        [
+            'filepath',
+            'name',
+        ]
+    )
+    available = list()
+    for d in directories:
+        try:
+            if not os.path.isdir(d):
+                continue
+            files = sorted(os.listdir(d))
+        except OSError:
+            continue
+        modules = [m for m in files if m.endswith(MODULE_SUFFIX)]
+        available.extend([AvailableModule(os.path.join(d, m), m[:-len(MODULE_SUFFIX)]) for m in modules])
+
+    return available
+
+
 def available_modules():
     obsolete = (
         'apache_cache',  # replaced by web_log
@@ -143,10 +171,17 @@ def available_modules():
         'unbound',  # rewritten in Go
     )
 
-    files = sorted(os.listdir(DIRS.modules))
-    modules = [m[:-len(MODULE_SUFFIX)] for m in files if m.endswith(MODULE_SUFFIX)]
-    avail = [m for m in modules if m not in obsolete]
-    return tuple(avail)
+    stock = [m for m in find_available_modules(DIRS.modules) if m.name not in obsolete]
+    user = find_available_modules(*DIRS.user_modules)
+
+    available, seen = list(), set()
+    for m in user + stock:
+        if m.name in seen:
+            continue
+        seen.add(m.name)
+        available.append(m)
+
+    return available
 
 
 AVAILABLE_MODULES = available_modules()
@@ -176,9 +211,8 @@ def multi_path_find(name, *paths):
     return str()
 
 
-def load_module(name):
-    abs_path = os.path.join(DIRS.modules, '{0}{1}'.format(name, MODULE_SUFFIX))
-    module = SourceFileLoader('pythond_' + name, abs_path)
+def load_module(name, filepath):
+    module = SourceFileLoader('pythond_' + name, filepath)
     if isinstance(module, types.ModuleType):
         return module
     return module.load_module()
@@ -331,12 +365,13 @@ class Job(threading.Thread):
 
 
 class ModuleSrc:
-    def __init__(self, name):
-        self.name = name
+    def __init__(self, m):
+        self.name = m.name
+        self.filepath = m.filepath
         self.src = None
 
     def load(self):
-        self.src = load_module(self.name)
+        self.src = load_module(self.name, self.filepath)
 
     def get(self, key):
         return getattr(self.src, key, None)
@@ -537,7 +572,8 @@ class Plugin:
         try:
             statuses = JobsStatuses().from_file(abs_path)
         except Exception as error:
-            self.log.warning("error on loading '{0}' : {1}".format(abs_path, repr(error)))
+            self.log.error("[{0}] config file invalid YAML format: {1}".format(
+                module_name, ' '.join([v.strip() for v in str(error).split('\n')])))
             return None
         self.log.debug("'{0}' is loaded".format(abs_path))
         return statuses
@@ -553,37 +589,38 @@ class Plugin:
         builder.min_update_every = self.min_update_every
 
         jobs = list()
-        for mod_name in self.modules_to_run:
-            if not self.config.is_module_enabled(mod_name):
-                self.log.info("[{0}] is disabled in the configuration file, skipping it".format(mod_name))
+        for m in self.modules_to_run:
+            if not self.config.is_module_enabled(m.name):
+                self.log.info("[{0}] is disabled in the configuration file, skipping it".format(m.name))
                 continue
 
-            src = ModuleSrc(mod_name)
+            src = ModuleSrc(m)
             try:
                 src.load()
             except Exception as error:
-                self.log.warning("[{0}] error on loading source : {1}, skipping it".format(mod_name, repr(error)))
+                self.log.warning("[{0}] error on loading source : {1}, skipping it".format(m.name, repr(error)))
                 continue
+            self.log.debug("[{0}] loaded module source : '{1}'".format(m.name, m.filepath))
 
             if not (src.service() and callable(src.service())):
-                self.log.warning("[{0}] has no callable Service object, skipping it".format(mod_name))
+                self.log.warning("[{0}] has no callable Service object, skipping it".format(m.name))
                 continue
 
-            if src.is_disabled_by_default() and not self.config.is_module_explicitly_enabled(mod_name):
-                self.log.info("[{0}] is disabled by default, skipping it".format(mod_name))
+            if src.is_disabled_by_default() and not self.config.is_module_explicitly_enabled(m.name):
+                self.log.info("[{0}] is disabled by default, skipping it".format(m.name))
                 continue
 
             builder.module_defaults = src.defaults()
-            configs = builder.build(mod_name)
+            configs = builder.build(m.name)
             if not configs:
-                self.log.info("[{0}] has no job configs, skipping it".format(mod_name))
+                self.log.info("[{0}] has no job configs, skipping it".format(m.name))
                 continue
 
             for config in configs:
                 config['job_name'] = re.sub(r'\s+', '_', config['job_name'])
                 config['override_name'] = re.sub(r'\s+', '_', config.pop('name'))
 
-                job = Job(src.service(), mod_name, config)
+                job = Job(src.service(), m.name, config)
 
                 was_previously_active = job_statuses and job_statuses.has(job.module_name, job.real_name)
                 if was_previously_active and job.autodetection_retry == 0:
@@ -811,6 +848,20 @@ def disable():
     exit(0)
 
 
+def get_modules_to_run(cmd):
+    if not cmd.modules_to_run:
+        return AVAILABLE_MODULES
+
+    modules_to_run, seen = list(), set()
+    for m in AVAILABLE_MODULES:
+        if m.name not in cmd.modules_to_run or m.name in seen:
+            continue
+        seen.add(m.name)
+        modules_to_run.append(m)
+
+    return modules_to_run
+
+
 def main():
     cmd = parse_command_line()
     log = PythonDLogger()
@@ -822,21 +873,22 @@ def main():
 
     log.info('using python v{0}'.format(PY_VERSION[0]))
 
-    unknown = set(cmd.modules_to_run) - set(AVAILABLE_MODULES)
+    if DIRS.locks and not cmd.nolock:
+        registry = FileLockRegistry(DIRS.locks)
+    else:
+        registry = DummyRegistry()
+
+    unique_avail_module_names = set([m.name for m in AVAILABLE_MODULES])
+    unknown = set(cmd.modules_to_run) - unique_avail_module_names
     if unknown:
         log.error('unknown modules : {0}'.format(sorted(list(unknown))))
-        guessed = guess_module(AVAILABLE_MODULES, *cmd.modules_to_run)
+        guessed = guess_module(unique_avail_module_names, *cmd.modules_to_run)
         if guessed:
             log.info('probably you meant : \n{0}'.format(pprint.pformat(guessed, width=1)))
         return
 
-    if DIRS.locks and not cmd.nolock:
-        registry = FileLockRegistry(DIRS.locks)
-    else:
-        registry = DummyRegistry()
-
     p = Plugin(
-        cmd.modules_to_run or AVAILABLE_MODULES,
+        get_modules_to_run(cmd),
         cmd.update_every,
         registry,
     )
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py
index c304ccec2..ed1b2e669 100644
--- a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py
+++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py
@@ -95,8 +95,9 @@ class SimpleService(PythonDLimitedLogger, object):
 
     @property
     def name(self):
-        if self.job_name and self.job_name != self.module_name:
-            return '_'.join([self.module_name, self.override_name or self.job_name])
+        name = self.override_name or self.job_name
+        if name and name != self.module_name:
+            return '_'.join([self.module_name, name])
         return self.module_name
 
     def actual_name(self):
diff --git a/collectors/python.d.plugin/python_modules/urllib3/_collections.py b/collectors/python.d.plugin/python_modules/urllib3/_collections.py
index c1d2fad36..2a6b3ec70 100644
--- a/collectors/python.d.plugin/python_modules/urllib3/_collections.py
+++ b/collectors/python.d.plugin/python_modules/urllib3/_collections.py
@@ -1,6 +1,11 @@
 # SPDX-License-Identifier: MIT
 from __future__ import absolute_import
-from collections import Mapping, MutableMapping
+
+try:
+    from collections import Mapping, MutableMapping
+except ImportError:
+    from collections.abc import Mapping, MutableMapping
+
 try:
     from threading import RLock
 except ImportError:  # Platform-specific: No threads available
diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py b/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py
index c0997b1a2..de5e49838 100644
--- a/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py
+++ b/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py
@@ -12,7 +12,13 @@ import select
 import socket
 import sys
 import time
-from collections import namedtuple, Mapping
+
+from collections import namedtuple
+
+try:
+    from collections import Mapping
+except ImportError:
+    from collections.abc import Mapping
 
 try:
     monotonic = time.monotonic
diff --git a/collectors/python.d.plugin/smartd_log/smartd_log.chart.py b/collectors/python.d.plugin/smartd_log/smartd_log.chart.py
index 75b8c8c40..dc4e95dec 100644
--- a/collectors/python.d.plugin/smartd_log/smartd_log.chart.py
+++ b/collectors/python.d.plugin/smartd_log/smartd_log.chart.py
@@ -630,6 +630,7 @@ class Service(SimpleService):
         self.exclude = configuration.get('exclude_disks', str()).split()
         self.disks = list()
         self.runs = 0
+        self.do_force_rescan = False
 
     def check(self):
         return self.scan() > 0
@@ -637,9 +638,10 @@ class Service(SimpleService):
     def get_data(self):
         self.runs += 1
 
-        if self.runs % DEF_RESCAN_INTERVAL == 0:
+        if self.do_force_rescan or self.runs % DEF_RESCAN_INTERVAL == 0:
             self.cleanup()
             self.scan()
+            self.do_force_rescan = False
 
         data = dict()
 
@@ -654,10 +656,12 @@ class Service(SimpleService):
 
             if changed is None:
                 disk.alive = False
+                self.do_force_rescan = True
                 continue
 
             if changed and disk.populate_attrs() is None:
                 disk.alive = False
+                self.do_force_rescan = True
                 continue
 
             data.update(disk.data())
diff --git a/collectors/python.d.plugin/smartd_log/smartd_log.conf b/collectors/python.d.plugin/smartd_log/smartd_log.conf
index 4f138d17a..6c01d953b 100644
--- a/collectors/python.d.plugin/smartd_log/smartd_log.conf
+++ b/collectors/python.d.plugin/smartd_log/smartd_log.conf
@@ -65,3 +65,11 @@
 #    exclude_disks: 'PATTERN1 PATTERN2'  # space separated patterns. If the pattern is in the drive name, the module will not collect data for it.
 #
 # ----------------------------------------------------------------------
+
+custom:
+  name: smartd_log
+  log_path: '/var/log/smartd/'
+
+debian:
+  name: smartd_log
+  log_path: '/var/lib/smartmontools/'
diff --git a/collectors/python.d.plugin/zscores/README.md b/collectors/python.d.plugin/zscores/README.md
index 7fb189f6a..4f84a6c1f 100644
--- a/collectors/python.d.plugin/zscores/README.md
+++ b/collectors/python.d.plugin/zscores/README.md
@@ -1,9 +1,7 @@
 <!--
----
 title: "zscores"
 description: "Use statistical anomaly detection to narrow your focus and shorten root cause analysis."
 custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/zscores/README.md
----
 -->
 
 # Z-Scores - basic anomaly detection for your key metrics and charts
@@ -143,4 +141,4 @@ per_chart_agg: 'mean' # 'absmax' will take the max absolute value across all dim
     - About ~50mb of ram (`apps.mem`) being continually used by the `python.d.plugin`.
 - If you activate this collector on a fresh node, it might take a little while to build up enough data to calculate a
   proper zscore. So until you actually have `train_secs` of available data the mean and stddev calculated will be subject
-  to more noise.
\ No newline at end of file
+  to more noise.