diff options
Diffstat (limited to 'collectors/python.d.plugin')
20 files changed, 164 insertions, 656 deletions
diff --git a/collectors/python.d.plugin/Makefile.am b/collectors/python.d.plugin/Makefile.am index 667f1627c..9377ebe8d 100644 --- a/collectors/python.d.plugin/Makefile.am +++ b/collectors/python.d.plugin/Makefile.am @@ -48,7 +48,6 @@ include bind_rndc/Makefile.inc include boinc/Makefile.inc include ceph/Makefile.inc include changefinder/Makefile.inc -include chrony/Makefile.inc include dockerd/Makefile.inc include dovecot/Makefile.inc include example/Makefile.inc @@ -71,7 +70,6 @@ include nginx_plus/Makefile.inc include nvidia_smi/Makefile.inc include nsd/Makefile.inc include ntpd/Makefile.inc -include ovpn_status_log/Makefile.inc include openldap/Makefile.inc include oracledb/Makefile.inc include postfix/Makefile.inc diff --git a/collectors/python.d.plugin/chrony/Makefile.inc b/collectors/python.d.plugin/chrony/Makefile.inc deleted file mode 100644 index 18a805b12..000000000 --- a/collectors/python.d.plugin/chrony/Makefile.inc +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# install these files -dist_python_DATA += chrony/chrony.chart.py -dist_pythonconfig_DATA += chrony/chrony.conf - -# do not install these files, but include them in the distribution -dist_noinst_DATA += chrony/README.md chrony/Makefile.inc - diff --git a/collectors/python.d.plugin/chrony/README.md b/collectors/python.d.plugin/chrony/README.md deleted file mode 100644 index 3093ec3f9..000000000 --- a/collectors/python.d.plugin/chrony/README.md +++ /dev/null @@ -1,61 +0,0 @@ -<!-- -title: "Chrony monitoring with Netdata" -custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/chrony/README.md -sidebar_label: "Chrony" ---> - -# Chrony monitoring with Netdata - -Monitors the precision and statistics of a local chronyd server, and produces: - -- frequency -- last offset -- RMS offset -- residual freq -- root delay -- root dispersion -- skew -- system time - -## Requirements - -Verify that user Netdata can execute `chronyc tracking`. If necessary, update `/etc/chrony.conf`, `cmdallow`. - -## Enable the collector - -The `chrony` collector is disabled by default. To enable it, use `edit-config` from the Netdata [config -directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`, to edit the `python.d.conf` file. - -```bash -cd /etc/netdata # Replace this path with your Netdata config directory, if different -sudo ./edit-config python.d.conf -``` - -Change the value of the `chrony` setting to `yes`. Save the file and restart the Netdata Agent with `sudo systemctl -restart netdata`, or the appropriate method for your system, to finish enabling the `chrony` collector. - -## Configuration - -Edit the `python.d/chrony.conf` configuration file using `edit-config` from the Netdata [config -directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`. - -```bash -cd /etc/netdata # Replace this path with your Netdata config directory, if different -sudo ./edit-config python.d/chrony.conf -``` - -Sample: - -```yaml -# data collection frequency: -update_every: 1 - -# chrony query command: -local: - command: 'chronyc -n tracking' -``` - -Save the file and restart the Netdata Agent with `sudo systemctl restart netdata`, or the [appropriate -method](/docs/configure/start-stop-restart.md) for your system, to finish configuring the `chrony` collector. - - diff --git a/collectors/python.d.plugin/chrony/chrony.chart.py b/collectors/python.d.plugin/chrony/chrony.chart.py deleted file mode 100644 index 91f725001..000000000 --- a/collectors/python.d.plugin/chrony/chrony.chart.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- -# Description: chrony netdata python.d module -# Author: Dominik Schloesser (domschl) -# SPDX-License-Identifier: GPL-3.0-or-later - -from bases.FrameworkServices.ExecutableService import ExecutableService - -# default module values (can be overridden per job in `config`) -update_every = 5 - -CHRONY_COMMAND = 'chronyc -n tracking' - -# charts order (can be overridden if you want less charts, or different order) -ORDER = [ - 'system', - 'offsets', - 'stratum', - 'root', - 'frequency', - 'residualfreq', - 'skew', -] - -CHARTS = { - 'system': { - 'options': [None, 'Chrony System Time Deltas', 'microseconds', 'system', 'chrony.system', 'area'], - 'lines': [ - ['timediff', 'system time', 'absolute', 1, 1000] - ] - }, - 'offsets': { - 'options': [None, 'Chrony System Time Offsets', 'microseconds', 'system', 'chrony.offsets', 'area'], - 'lines': [ - ['lastoffset', 'last offset', 'absolute', 1, 1000], - ['rmsoffset', 'RMS offset', 'absolute', 1, 1000] - ] - }, - 'stratum': { - 'options': [None, 'Chrony Stratum', 'stratum', 'root', 'chrony.stratum', 'line'], - 'lines': [ - ['stratum', None, 'absolute', 1, 1] - ] - }, - 'root': { - 'options': [None, 'Chrony Root Delays', 'milliseconds', 'root', 'chrony.root', 'line'], - 'lines': [ - ['rootdelay', 'delay', 'absolute', 1, 1000000], - ['rootdispersion', 'dispersion', 'absolute', 1, 1000000] - ] - }, - 'frequency': { - 'options': [None, 'Chrony Frequency', 'ppm', 'frequencies', 'chrony.frequency', 'area'], - 'lines': [ - ['frequency', None, 'absolute', 1, 1000] - ] - }, - 'residualfreq': { - 'options': [None, 'Chrony Residual frequency', 'ppm', 'frequencies', 'chrony.residualfreq', 'area'], - 'lines': [ - ['residualfreq', 'residual frequency', 'absolute', 1, 1000] - ] - }, - 'skew': { - 'options': [None, 'Chrony Skew, error bound on frequency', 'ppm', 'frequencies', 'chrony.skew', 'area'], - 'lines': [ - ['skew', None, 'absolute', 1, 1000] - ] - } -} - -CHRONY = [ - ('Frequency', 'frequency', 1e3), - ('Last offset', 'lastoffset', 1e9), - ('RMS offset', 'rmsoffset', 1e9), - ('Residual freq', 'residualfreq', 1e3), - ('Root delay', 'rootdelay', 1e9), - ('Root dispersion', 'rootdispersion', 1e9), - ('Skew', 'skew', 1e3), - ('Stratum', 'stratum', 1), - ('System time', 'timediff', 1e9) -] - - -class Service(ExecutableService): - def __init__(self, configuration=None, name=None): - ExecutableService.__init__( - self, configuration=configuration, name=name) - self.order = ORDER - self.definitions = CHARTS - self.command = CHRONY_COMMAND - - def _get_data(self): - """ - Format data received from shell command - :return: dict - """ - raw_data = self._get_raw_data() - if not raw_data: - return None - - raw_data = (line.split(':', 1) for line in raw_data) - parsed, data = dict(), dict() - - for line in raw_data: - try: - key, value = (l.strip() for l in line) - except ValueError: - continue - if value: - parsed[key] = value.split()[0] - - for key, dim_id, multiplier in CHRONY: - try: - data[dim_id] = int(float(parsed[key]) * multiplier) - except (KeyError, ValueError): - continue - - return data or None diff --git a/collectors/python.d.plugin/chrony/chrony.conf b/collectors/python.d.plugin/chrony/chrony.conf deleted file mode 100644 index fd95519b5..000000000 --- a/collectors/python.d.plugin/chrony/chrony.conf +++ /dev/null @@ -1,77 +0,0 @@ -# netdata python.d.plugin configuration for chrony -# -# This file is in YaML format. Generally the format is: -# -# name: value -# -# There are 2 sections: -# - global variables -# - one or more JOBS -# -# JOBS allow you to collect values from multiple sources. -# Each source will have its own set of charts. -# -# JOB parameters have to be indented (using spaces only, example below). - -# ---------------------------------------------------------------------- -# Global Variables -# These variables set the defaults for all JOBs, however each JOB -# may define its own, overriding the defaults. - -# update_every sets the default data collection frequency. -# If unset, the python.d.plugin default is used. -update_every: 5 - -# priority controls the order of charts at the netdata dashboard. -# Lower numbers move the charts towards the top of the page. -# If unset, the default for python.d.plugin is used. -# priority: 60000 - -# penalty indicates whether to apply penalty to update_every in case of failures. -# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes. -# penalty: yes - -# autodetection_retry sets the job re-check interval in seconds. -# The job is not deleted if check fails. -# Attempts to start the job are made once every autodetection_retry. -# This feature is disabled by default. -# autodetection_retry: 0 - -# ---------------------------------------------------------------------- -# JOBS (data collection sources) -# -# The default JOBS share the same *name*. JOBS with the same name -# are mutually exclusive. Only one of them will be allowed running at -# any time. This allows autodetection to try several alternatives and -# pick the one that works. -# -# Any number of jobs is supported. -# -# All python.d.plugin JOBS (for all its modules) support a set of -# predefined parameters. These are: -# -# job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# penalty: yes # the JOB's penalty -# autodetection_retry: 0 # the JOB's re-check interval in seconds -# -# Additionally to the above, chrony also supports the following: -# -# command: 'chrony tracking' # the command to run -# - -# ---------------------------------------------------------------------- -# REQUIRED chrony CONFIGURATION -# -# netdata will query chrony as user netdata. -# verify that user netdata is allowed to call 'chronyc tracking' -# Check cmdallow in chrony.conf -# ---------------------------------------------------------------------- -# AUTO-DETECTION JOBS - -local: - command: 'chronyc -n tracking' diff --git a/collectors/python.d.plugin/haproxy/README.md b/collectors/python.d.plugin/haproxy/README.md index 4eb962e24..f16e7258e 100644 --- a/collectors/python.d.plugin/haproxy/README.md +++ b/collectors/python.d.plugin/haproxy/README.md @@ -9,30 +9,32 @@ sidebar_label: "HAProxy" Monitors frontend and backend metrics such as bytes in, bytes out, sessions current, sessions in queue current. And health metrics such as backend servers status (server check should be used). -Plugin can obtain data from url **OR** unix socket. +Plugin can obtain data from URL or Unix socket. -**Requirement:** -Socket MUST be readable AND writable by the `netdata` user. +Requirement: + +- Socket must be readable and writable by the `netdata` user. +- URL must have `stats uri <path>` present in the haproxy config, otherwise you will get HTTP 503 in the haproxy logs. It produces: -1. **Frontend** family charts +1. **Frontend** family charts - - Kilobytes in/s - - Kilobytes out/s - - Sessions current - - Sessions in queue current + - Kilobytes in/s + - Kilobytes out/s + - Sessions current + - Sessions in queue current -2. **Backend** family charts +2. **Backend** family charts - - Kilobytes in/s - - Kilobytes out/s - - Sessions current - - Sessions in queue current + - Kilobytes in/s + - Kilobytes out/s + - Sessions current + - Sessions in queue current -3. **Health** chart +3. **Health** chart - - number of failed servers for every backend (in DOWN state) + - number of failed servers for every backend (in DOWN state) ## Configuration @@ -48,20 +50,18 @@ Sample: ```yaml via_url: - user : 'username' # ONLY IF stats auth is used - pass : 'password' # # ONLY IF stats auth is used - url : 'http://ip.address:port/url;csv;norefresh' + user: 'username' # ONLY IF stats auth is used + pass: 'password' # # ONLY IF stats auth is used + url: 'http://ip.address:port/url;csv;norefresh' ``` OR ```yaml via_socket: - socket : 'path/to/haproxy/sock' + socket: 'path/to/haproxy/sock' ``` If no configuration is given, module will fail to run. --- - - diff --git a/collectors/python.d.plugin/mongodb/mongodb.chart.py b/collectors/python.d.plugin/mongodb/mongodb.chart.py index bec94d3ef..5e8fec834 100644 --- a/collectors/python.d.plugin/mongodb/mongodb.chart.py +++ b/collectors/python.d.plugin/mongodb/mongodb.chart.py @@ -10,7 +10,7 @@ from datetime import datetime from sys import exc_info try: - from pymongo import MongoClient, ASCENDING, DESCENDING + from pymongo import MongoClient, ASCENDING, DESCENDING, version_tuple from pymongo.errors import PyMongoError PYMONGO = True @@ -750,7 +750,7 @@ class Service(SimpleService): CONN_PARAM_HOST: conf.get(CONN_PARAM_HOST, DEFAULT_HOST), CONN_PARAM_PORT: conf.get(CONN_PARAM_PORT, DEFAULT_PORT), } - if hasattr(MongoClient, 'server_selection_timeout'): + if hasattr(MongoClient, 'server_selection_timeout') or version_tuple[0] >= 4: params[CONN_PARAM_SERVER_SELECTION_TIMEOUT_MS] = conf.get('timeout', DEFAULT_TIMEOUT) params.update(self.build_ssl_connection_params()) diff --git a/collectors/python.d.plugin/ovpn_status_log/Makefile.inc b/collectors/python.d.plugin/ovpn_status_log/Makefile.inc deleted file mode 100644 index 1fbc506d6..000000000 --- a/collectors/python.d.plugin/ovpn_status_log/Makefile.inc +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# install these files -dist_python_DATA += ovpn_status_log/ovpn_status_log.chart.py -dist_pythonconfig_DATA += ovpn_status_log/ovpn_status_log.conf - -# do not install these files, but include them in the distribution -dist_noinst_DATA += ovpn_status_log/README.md ovpn_status_log/Makefile.inc - diff --git a/collectors/python.d.plugin/ovpn_status_log/README.md b/collectors/python.d.plugin/ovpn_status_log/README.md deleted file mode 100644 index 374d1910e..000000000 --- a/collectors/python.d.plugin/ovpn_status_log/README.md +++ /dev/null @@ -1,50 +0,0 @@ -<!-- -title: "OpenVPN monitoring with Netdata" -custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/ovpn_status_log/README.md -sidebar_label: "OpenVPN" ---> - -# OpenVPN monitoring with Netdata - -Parses server log files and provides summary (client, traffic) metrics. - -## Requirements - -- If you are running multiple OpenVPN instances out of the same directory, MAKE SURE TO EDIT DIRECTIVES which create output files - so that multiple instances do not overwrite each other's output files. - -- Make sure NETDATA USER CAN READ openvpn-status.log - -- Update_every interval MUST MATCH interval on which OpenVPN writes operational status to log file. - -It produces: - -1. **Users** OpenVPN active users - - - users - -2. **Traffic** OpenVPN overall bandwidth usage in kilobit/s - - - in - - out - -## Configuration - -Edit the `python.d/ovpn_status_log.conf` configuration file using `edit-config` from the Netdata [config -directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`. - -```bash -cd /etc/netdata # Replace this path with your Netdata config directory, if different -sudo ./edit-config python.d/ovpn_status_log.conf -``` - -Sample: - -```yaml -default - log_path : '/var/log/openvpn-status.log' -``` - ---- - - diff --git a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.chart.py b/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.chart.py deleted file mode 100644 index cfc87be36..000000000 --- a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.chart.py +++ /dev/null @@ -1,136 +0,0 @@ -# -*- coding: utf-8 -*- -# Description: openvpn status log netdata python.d module -# Author: ilyam8 -# SPDX-License-Identifier: GPL-3.0-or-later - -import re - -from bases.FrameworkServices.SimpleService import SimpleService - -update_every = 10 - -ORDER = [ - 'users', - 'traffic', -] - -CHARTS = { - 'users': { - 'options': [None, 'OpenVPN Active Users', 'active users', 'users', 'openvpn_status.users', 'line'], - 'lines': [ - ['users', None, 'absolute'], - ] - }, - 'traffic': { - 'options': [None, 'OpenVPN Traffic', 'KiB/s', 'traffic', 'openvpn_status.traffic', 'area'], - 'lines': [ - ['bytes_in', 'in', 'incremental', 1, 1 << 10], - ['bytes_out', 'out', 'incremental', -1, 1 << 10] - ] - } -} - -TLS_REGEX = re.compile( - r'(?:[0-9a-f]+:[0-9a-f:]+|(?:\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?)) (?P<bytes_in>\d+) (?P<bytes_out>\d+)' -) -STATIC_KEY_REGEX = re.compile( - r'TCP/[A-Z]+ (?P<direction>(?:read|write)) bytes,(?P<bytes>\d+)' -) - - -class Service(SimpleService): - def __init__(self, configuration=None, name=None): - SimpleService.__init__(self, configuration=configuration, name=name) - self.order = ORDER - self.definitions = CHARTS - self.log_path = self.configuration.get('log_path') - self.regex = { - 'tls': TLS_REGEX, - 'static_key': STATIC_KEY_REGEX - } - - def check(self): - if not (self.log_path and isinstance(self.log_path, str)): - self.error("'log_path' is not defined") - return False - - data = self._get_raw_data() - if not data: - self.error('Make sure that the openvpn status log file exists and netdata has permission to read it') - return None - - found = None - for row in data: - if 'ROUTING' in row: - self.get_data = self.get_data_tls - found = True - break - elif 'STATISTICS' in row: - self.get_data = self.get_data_static_key - found = True - break - if found: - return True - self.error('Failed to parse openvpn log file') - return False - - def _get_raw_data(self): - """ - Open log file - :return: str - """ - - try: - with open(self.log_path) as log: - raw_data = log.readlines() or None - except OSError: - return None - else: - return raw_data - - def get_data_static_key(self): - """ - Parse openvpn-status log file. - """ - - raw_data = self._get_raw_data() - if not raw_data: - return None - - data = dict(bytes_in=0, bytes_out=0) - - for row in raw_data: - match = self.regex['static_key'].search(row) - if match: - match = match.groupdict() - if match['direction'] == 'read': - data['bytes_in'] += int(match['bytes']) - else: - data['bytes_out'] += int(match['bytes']) - - return data or None - - def get_data_tls(self): - """ - Parse openvpn-status log file. - """ - - raw_data = self._get_raw_data() - if not raw_data: - return None - - data = dict(users=0, bytes_in=0, bytes_out=0) - for row in raw_data: - columns = row.split(',') if ',' in row else row.split() - if 'UNDEF' in columns: - # see https://openvpn.net/archive/openvpn-users/2004-08/msg00116.html - continue - - match = self.regex['tls'].search(' '.join(columns)) - if match: - match = match.groupdict() - data['users'] += 1 - data['bytes_in'] += int(match['bytes_in']) - data['bytes_out'] += int(match['bytes_out']) - - return data or None diff --git a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.conf b/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.conf deleted file mode 100644 index 1d71f6b8e..000000000 --- a/collectors/python.d.plugin/ovpn_status_log/ovpn_status_log.conf +++ /dev/null @@ -1,97 +0,0 @@ -# netdata python.d.plugin configuration for openvpn status log -# -# This file is in YaML format. Generally the format is: -# -# name: value -# -# There are 2 sections: -# - global variables -# - one or more JOBS -# -# JOBS allow you to collect values from multiple sources. -# Each source will have its own set of charts. -# -# JOB parameters have to be indented (using spaces only, example below). - -# ---------------------------------------------------------------------- -# Global Variables -# These variables set the defaults for all JOBs, however each JOB -# may define its own, overriding the defaults. - -# update_every sets the default data collection frequency. -# If unset, the python.d.plugin default is used. -# update_every: 1 - -# priority controls the order of charts at the netdata dashboard. -# Lower numbers move the charts towards the top of the page. -# If unset, the default for python.d.plugin is used. -# priority: 60000 - -# penalty indicates whether to apply penalty to update_every in case of failures. -# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes. -# penalty: yes - -# autodetection_retry sets the job re-check interval in seconds. -# The job is not deleted if check fails. -# Attempts to start the job are made once every autodetection_retry. -# This feature is disabled by default. -# autodetection_retry: 0 - -# ---------------------------------------------------------------------- -# JOBS (data collection sources) -# -# The default JOBS share the same *name*. JOBS with the same name -# are mutually exclusive. Only one of them will be allowed running at -# any time. This allows autodetection to try several alternatives and -# pick the one that works. -# -# Any number of jobs is supported. -# -# All python.d.plugin JOBS (for all its modules) support a set of -# predefined parameters. These are: -# -# job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# penalty: yes # the JOB's penalty -# autodetection_retry: 0 # the JOB's re-check interval in seconds -# -# Additionally to the above, openvpn status log also supports the following: -# -# log_path: 'PATH' # the path to openvpn status log file -# -# ---------------------------------------------------------------------- -# AUTO-DETECTION JOBS -# only one of them will run (they have the same name) -# -# IMPORTANT information -# -# 1. If you are running multiple OpenVPN instances out of the same directory, MAKE SURE TO EDIT DIRECTIVES which create output files -# so that multiple instances do not overwrite each other's output files. -# 2. Make sure NETDATA USER CAN READ openvpn-status.log -# -# * cd into directory with openvpn-status.log and run the following commands as root -# * #chown :netdata openvpn-status.log && chmod 640 openvpn-status.log -# * To check permission and group membership run -# * #ls -l openvpn-status.log -# -rw-r----- 1 root netdata 359 dec 21 21:22 openvpn-status.log -# -# 3. Update_every interval MUST MATCH interval on which OpenVPN writes operational status to log file. -# If its not true traffic chart WILL DISPLAY WRONG values -# -# Default OpenVPN update interval is 10 second on Debian 8 -# # ps -C openvpn -o command= -# /usr/sbin/openvpn --daemon ovpn-server --status /run/openvpn/server.status 10 --cd /etc/openvpn --config /etc/openvpn/server.conf -# -# 4. Confirm status is configured in your OpenVPN configuration. -# * Open OpenVPN config in an editor (e.g. sudo nano /etc/openvpn/default.conf) -# * Confirm status is enabled with below: -# status /var/log/openvpn-status.log -# -#default: -# log_path: '/var/log/openvpn-status.log' -# -# ---------------------------------------------------------------------- diff --git a/collectors/python.d.plugin/postgres/README.md b/collectors/python.d.plugin/postgres/README.md index 224b76ff5..7acb9a7a9 100644 --- a/collectors/python.d.plugin/postgres/README.md +++ b/collectors/python.d.plugin/postgres/README.md @@ -6,6 +6,9 @@ sidebar_label: "PostgreSQL" # PostgreSQL monitoring with Netdata +> **Warning**: This module is deprecated and will be deleted in v1.37.0. +> Use [go.d/postgres](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/postgres). + Collects database health and performance metrics. ## Requirements @@ -97,7 +100,8 @@ cd /etc/netdata # Replace this path with your Netdata config directory, if dif sudo ./edit-config python.d/postgres.conf ``` -When no configuration file is found, the module tries to connect to TCP/IP socket: `localhost:5432`. +When no configuration file is found, the module tries to connect to TCP/IP socket: `localhost:5432` with the +following collection jobs. ```yaml socket: @@ -113,6 +117,29 @@ tcp: port : 5432 ``` +**Note**: Every job collection must have a unique identifier. In cases that you monitor multiple DBs, every +job must have it's own name. Use a mnemonic of your preference (e.g us_east_db, us_east_tcp) + +## Troubleshooting + +To troubleshoot issues with the `postgres` collector, run the `python.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +First, navigate to your plugins directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on your +system, open `netdata.conf` and look for the setting `plugins directory`. Once you're in the plugin's directory, switch +to the `netdata` user. + +```bash +cd /usr/libexec/netdata/plugins.d/ +sudo su -s /bin/bash netdata +``` + +You can now run the `python.d.plugin` to debug the collector: + +```bash +./python.d.plugin postgres debug trace +``` + --- diff --git a/collectors/python.d.plugin/python.d.conf b/collectors/python.d.plugin/python.d.conf index af58b451c..72e20fcd3 100644 --- a/collectors/python.d.plugin/python.d.conf +++ b/collectors/python.d.plugin/python.d.conf @@ -25,73 +25,48 @@ gc_run: yes # Garbage collection interval in seconds. Default is 300. gc_interval: 300 -# apache: yes - -# apache_cache has been replaced by web_log # adaptec_raid: yes # alarms: yes # am2320: yes # anomalies: no -apache_cache: no # beanstalk: yes # bind_rndc: yes # boinc: yes # ceph: yes -chrony: no # changefinder: no -# couchdb: yes -# dns_query_time: yes -# dnsdist: yes # dockerd: yes # dovecot: yes -# elasticsearch: yes -# energid: yes # this is just an example example: no # exim: yes # fail2ban: yes -# freeradius: yes # gearman: yes go_expvar: no -# gunicorn_log has been replaced by web_log -gunicorn_log: no # haproxy: yes # hddtemp: yes -# httpcheck: yes hpssa: no # icecast: yes # ipfs: yes -# isc_dhcpd: yes # litespeed: yes logind: no # megacli: yes # memcached: yes # mongodb: yes # monit: yes -# mysql: yes -# nginx: yes # nginx_plus: yes # nvidia_smi: yes - -# nginx_log has been replaced by web_log -nginx_log: no # nsd: yes # ntpd: yes # openldap: yes # oracledb: yes -# ovpn_status_log: yes -# phpfpm: yes -# portcheck: yes # postfix: yes # postgres: yes -# powerdns: yes # proxysql: yes # puppet: yes # rabbitmq: yes -# redis: yes # rethinkdbs: yes # retroshare: yes # riakkv: yes @@ -107,5 +82,4 @@ nginx_log: no # uwsgi: yes # varnish: yes # w1sensor: yes -# web_log: yes # zscores: no diff --git a/collectors/python.d.plugin/python.d.plugin.in b/collectors/python.d.plugin/python.d.plugin.in index b943f3a20..c04cb3ff0 100644 --- a/collectors/python.d.plugin/python.d.plugin.in +++ b/collectors/python.d.plugin/python.d.plugin.in @@ -31,8 +31,8 @@ import os import pprint import re import sys -import time import threading +import time import types try: @@ -50,6 +50,7 @@ else: ENV_NETDATA_USER_CONFIG_DIR = 'NETDATA_USER_CONFIG_DIR' ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR' ENV_NETDATA_PLUGINS_DIR = 'NETDATA_PLUGINS_DIR' +ENV_NETDATA_USER_PLUGINS_DIRS = 'NETDATA_USER_PLUGINS_DIRS' ENV_NETDATA_LIB_DIR = 'NETDATA_LIB_DIR' ENV_NETDATA_UPDATE_EVERY = 'NETDATA_UPDATE_EVERY' ENV_NETDATA_LOCK_DIR = 'NETDATA_LOCK_DIR' @@ -99,6 +100,9 @@ def dirs(): modules_user_config = os.path.join(plugin_user_config, 'python.d') modules_stock_config = os.path.join(plugin_stock_config, 'python.d') modules = os.path.abspath(pluginsd + '/../python.d') + user_modules = [os.path.join(p, 'python.d') for p in + os.getenv(ENV_NETDATA_USER_PLUGINS_DIRS, "").split(" ") if + p] Dirs = collections.namedtuple( 'Dirs', @@ -108,6 +112,7 @@ def dirs(): 'modules_user_config', 'modules_stock_config', 'modules', + 'user_modules', 'var_lib', 'locks', ] @@ -118,6 +123,7 @@ def dirs(): modules_user_config, modules_stock_config, modules, + user_modules, var_lib, locks, ) @@ -130,6 +136,28 @@ IS_ATTY = sys.stdout.isatty() MODULE_SUFFIX = '.chart.py' +def find_available_modules(*directories): + AvailableModule = collections.namedtuple( + 'AvailableModule', + [ + 'filepath', + 'name', + ] + ) + available = list() + for d in directories: + try: + if not os.path.isdir(d): + continue + files = sorted(os.listdir(d)) + except OSError: + continue + modules = [m for m in files if m.endswith(MODULE_SUFFIX)] + available.extend([AvailableModule(os.path.join(d, m), m[:-len(MODULE_SUFFIX)]) for m in modules]) + + return available + + def available_modules(): obsolete = ( 'apache_cache', # replaced by web_log @@ -143,10 +171,17 @@ def available_modules(): 'unbound', # rewritten in Go ) - files = sorted(os.listdir(DIRS.modules)) - modules = [m[:-len(MODULE_SUFFIX)] for m in files if m.endswith(MODULE_SUFFIX)] - avail = [m for m in modules if m not in obsolete] - return tuple(avail) + stock = [m for m in find_available_modules(DIRS.modules) if m.name not in obsolete] + user = find_available_modules(*DIRS.user_modules) + + available, seen = list(), set() + for m in user + stock: + if m.name in seen: + continue + seen.add(m.name) + available.append(m) + + return available AVAILABLE_MODULES = available_modules() @@ -176,9 +211,8 @@ def multi_path_find(name, *paths): return str() -def load_module(name): - abs_path = os.path.join(DIRS.modules, '{0}{1}'.format(name, MODULE_SUFFIX)) - module = SourceFileLoader('pythond_' + name, abs_path) +def load_module(name, filepath): + module = SourceFileLoader('pythond_' + name, filepath) if isinstance(module, types.ModuleType): return module return module.load_module() @@ -331,12 +365,13 @@ class Job(threading.Thread): class ModuleSrc: - def __init__(self, name): - self.name = name + def __init__(self, m): + self.name = m.name + self.filepath = m.filepath self.src = None def load(self): - self.src = load_module(self.name) + self.src = load_module(self.name, self.filepath) def get(self, key): return getattr(self.src, key, None) @@ -537,7 +572,8 @@ class Plugin: try: statuses = JobsStatuses().from_file(abs_path) except Exception as error: - self.log.warning("error on loading '{0}' : {1}".format(abs_path, repr(error))) + self.log.error("[{0}] config file invalid YAML format: {1}".format( + module_name, ' '.join([v.strip() for v in str(error).split('\n')]))) return None self.log.debug("'{0}' is loaded".format(abs_path)) return statuses @@ -553,37 +589,38 @@ class Plugin: builder.min_update_every = self.min_update_every jobs = list() - for mod_name in self.modules_to_run: - if not self.config.is_module_enabled(mod_name): - self.log.info("[{0}] is disabled in the configuration file, skipping it".format(mod_name)) + for m in self.modules_to_run: + if not self.config.is_module_enabled(m.name): + self.log.info("[{0}] is disabled in the configuration file, skipping it".format(m.name)) continue - src = ModuleSrc(mod_name) + src = ModuleSrc(m) try: src.load() except Exception as error: - self.log.warning("[{0}] error on loading source : {1}, skipping it".format(mod_name, repr(error))) + self.log.warning("[{0}] error on loading source : {1}, skipping it".format(m.name, repr(error))) continue + self.log.debug("[{0}] loaded module source : '{1}'".format(m.name, m.filepath)) if not (src.service() and callable(src.service())): - self.log.warning("[{0}] has no callable Service object, skipping it".format(mod_name)) + self.log.warning("[{0}] has no callable Service object, skipping it".format(m.name)) continue - if src.is_disabled_by_default() and not self.config.is_module_explicitly_enabled(mod_name): - self.log.info("[{0}] is disabled by default, skipping it".format(mod_name)) + if src.is_disabled_by_default() and not self.config.is_module_explicitly_enabled(m.name): + self.log.info("[{0}] is disabled by default, skipping it".format(m.name)) continue builder.module_defaults = src.defaults() - configs = builder.build(mod_name) + configs = builder.build(m.name) if not configs: - self.log.info("[{0}] has no job configs, skipping it".format(mod_name)) + self.log.info("[{0}] has no job configs, skipping it".format(m.name)) continue for config in configs: config['job_name'] = re.sub(r'\s+', '_', config['job_name']) config['override_name'] = re.sub(r'\s+', '_', config.pop('name')) - job = Job(src.service(), mod_name, config) + job = Job(src.service(), m.name, config) was_previously_active = job_statuses and job_statuses.has(job.module_name, job.real_name) if was_previously_active and job.autodetection_retry == 0: @@ -811,6 +848,20 @@ def disable(): exit(0) +def get_modules_to_run(cmd): + if not cmd.modules_to_run: + return AVAILABLE_MODULES + + modules_to_run, seen = list(), set() + for m in AVAILABLE_MODULES: + if m.name not in cmd.modules_to_run or m.name in seen: + continue + seen.add(m.name) + modules_to_run.append(m) + + return modules_to_run + + def main(): cmd = parse_command_line() log = PythonDLogger() @@ -822,21 +873,22 @@ def main(): log.info('using python v{0}'.format(PY_VERSION[0])) - unknown = set(cmd.modules_to_run) - set(AVAILABLE_MODULES) + if DIRS.locks and not cmd.nolock: + registry = FileLockRegistry(DIRS.locks) + else: + registry = DummyRegistry() + + unique_avail_module_names = set([m.name for m in AVAILABLE_MODULES]) + unknown = set(cmd.modules_to_run) - unique_avail_module_names if unknown: log.error('unknown modules : {0}'.format(sorted(list(unknown)))) - guessed = guess_module(AVAILABLE_MODULES, *cmd.modules_to_run) + guessed = guess_module(unique_avail_module_names, *cmd.modules_to_run) if guessed: log.info('probably you meant : \n{0}'.format(pprint.pformat(guessed, width=1))) return - if DIRS.locks and not cmd.nolock: - registry = FileLockRegistry(DIRS.locks) - else: - registry = DummyRegistry() - p = Plugin( - cmd.modules_to_run or AVAILABLE_MODULES, + get_modules_to_run(cmd), cmd.update_every, registry, ) diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py index c304ccec2..ed1b2e669 100644 --- a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py @@ -95,8 +95,9 @@ class SimpleService(PythonDLimitedLogger, object): @property def name(self): - if self.job_name and self.job_name != self.module_name: - return '_'.join([self.module_name, self.override_name or self.job_name]) + name = self.override_name or self.job_name + if name and name != self.module_name: + return '_'.join([self.module_name, name]) return self.module_name def actual_name(self): diff --git a/collectors/python.d.plugin/python_modules/urllib3/_collections.py b/collectors/python.d.plugin/python_modules/urllib3/_collections.py index c1d2fad36..2a6b3ec70 100644 --- a/collectors/python.d.plugin/python_modules/urllib3/_collections.py +++ b/collectors/python.d.plugin/python_modules/urllib3/_collections.py @@ -1,6 +1,11 @@ # SPDX-License-Identifier: MIT from __future__ import absolute_import -from collections import Mapping, MutableMapping + +try: + from collections import Mapping, MutableMapping +except ImportError: + from collections.abc import Mapping, MutableMapping + try: from threading import RLock except ImportError: # Platform-specific: No threads available diff --git a/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py b/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py index c0997b1a2..de5e49838 100644 --- a/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py +++ b/collectors/python.d.plugin/python_modules/urllib3/util/selectors.py @@ -12,7 +12,13 @@ import select import socket import sys import time -from collections import namedtuple, Mapping + +from collections import namedtuple + +try: + from collections import Mapping +except ImportError: + from collections.abc import Mapping try: monotonic = time.monotonic diff --git a/collectors/python.d.plugin/smartd_log/smartd_log.chart.py b/collectors/python.d.plugin/smartd_log/smartd_log.chart.py index 75b8c8c40..dc4e95dec 100644 --- a/collectors/python.d.plugin/smartd_log/smartd_log.chart.py +++ b/collectors/python.d.plugin/smartd_log/smartd_log.chart.py @@ -630,6 +630,7 @@ class Service(SimpleService): self.exclude = configuration.get('exclude_disks', str()).split() self.disks = list() self.runs = 0 + self.do_force_rescan = False def check(self): return self.scan() > 0 @@ -637,9 +638,10 @@ class Service(SimpleService): def get_data(self): self.runs += 1 - if self.runs % DEF_RESCAN_INTERVAL == 0: + if self.do_force_rescan or self.runs % DEF_RESCAN_INTERVAL == 0: self.cleanup() self.scan() + self.do_force_rescan = False data = dict() @@ -654,10 +656,12 @@ class Service(SimpleService): if changed is None: disk.alive = False + self.do_force_rescan = True continue if changed and disk.populate_attrs() is None: disk.alive = False + self.do_force_rescan = True continue data.update(disk.data()) diff --git a/collectors/python.d.plugin/smartd_log/smartd_log.conf b/collectors/python.d.plugin/smartd_log/smartd_log.conf index 4f138d17a..6c01d953b 100644 --- a/collectors/python.d.plugin/smartd_log/smartd_log.conf +++ b/collectors/python.d.plugin/smartd_log/smartd_log.conf @@ -65,3 +65,11 @@ # exclude_disks: 'PATTERN1 PATTERN2' # space separated patterns. If the pattern is in the drive name, the module will not collect data for it. # # ---------------------------------------------------------------------- + +custom: + name: smartd_log + log_path: '/var/log/smartd/' + +debian: + name: smartd_log + log_path: '/var/lib/smartmontools/' diff --git a/collectors/python.d.plugin/zscores/README.md b/collectors/python.d.plugin/zscores/README.md index 7fb189f6a..4f84a6c1f 100644 --- a/collectors/python.d.plugin/zscores/README.md +++ b/collectors/python.d.plugin/zscores/README.md @@ -1,9 +1,7 @@ <!-- ---- title: "zscores" description: "Use statistical anomaly detection to narrow your focus and shorten root cause analysis." custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/zscores/README.md ---- --> # Z-Scores - basic anomaly detection for your key metrics and charts @@ -143,4 +141,4 @@ per_chart_agg: 'mean' # 'absmax' will take the max absolute value across all dim - About ~50mb of ram (`apps.mem`) being continually used by the `python.d.plugin`. - If you activate this collector on a fresh node, it might take a little while to build up enough data to calculate a proper zscore. So until you actually have `train_secs` of available data the mean and stddev calculated will be subject - to more noise.
\ No newline at end of file + to more noise. |