diff options
Diffstat (limited to '')
| -rw-r--r-- | collectors/charts.d.plugin/apcupsd/README.md (renamed from python.d/python_modules/bases/FrameworkServices/__init__.py) | 0 | ||||
| -rw-r--r-- | collectors/charts.d.plugin/opensips/README.md (renamed from python.d/python_modules/bases/__init__.py) | 0 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py (renamed from python.d/python_modules/bases/FrameworkServices/ExecutableService.py) | 9 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/FrameworkServices/LogService.py (renamed from python.d/python_modules/bases/FrameworkServices/LogService.py) | 2 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py (renamed from python.d/python_modules/bases/FrameworkServices/MySQLService.py) | 1 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py (renamed from python.d/python_modules/bases/FrameworkServices/SimpleService.py) | 13 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py (renamed from python.d/python_modules/bases/FrameworkServices/SocketService.py) | 64 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py (renamed from python.d/python_modules/bases/FrameworkServices/UrlService.py) | 32 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/charts.py (renamed from python.d/python_modules/bases/charts.py) | 18 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/collection.py (renamed from python.d/python_modules/bases/collection.py) | 1 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/loaders.py (renamed from python.d/python_modules/bases/loaders.py) | 21 | ||||
| -rw-r--r-- | collectors/python.d.plugin/python_modules/bases/loggers.py (renamed from python.d/python_modules/bases/loggers.py) | 1 |
12 files changed, 132 insertions, 30 deletions
diff --git a/python.d/python_modules/bases/FrameworkServices/__init__.py b/collectors/charts.d.plugin/apcupsd/README.md index e69de29bb..e69de29bb 100644 --- a/python.d/python_modules/bases/FrameworkServices/__init__.py +++ b/collectors/charts.d.plugin/apcupsd/README.md diff --git a/python.d/python_modules/bases/__init__.py b/collectors/charts.d.plugin/opensips/README.md index e69de29bb..e69de29bb 100644 --- a/python.d/python_modules/bases/__init__.py +++ b/collectors/charts.d.plugin/opensips/README.md diff --git a/python.d/python_modules/bases/FrameworkServices/ExecutableService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py index a71f2bfd2..72f9ff714 100644 --- a/python.d/python_modules/bases/FrameworkServices/ExecutableService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py @@ -2,6 +2,7 @@ # Description: # Author: Pawel Krupa (paulfantom) # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later import os @@ -16,15 +17,15 @@ class ExecutableService(SimpleService): SimpleService.__init__(self, configuration=configuration, name=name) self.command = None - def _get_raw_data(self, stderr=False): + def _get_raw_data(self, stderr=False, command=None): """ Get raw data from executed command :return: <list> """ try: - p = Popen(self.command, stdout=PIPE, stderr=PIPE) + p = Popen(command if command else self.command, stdout=PIPE, stderr=PIPE) except Exception as error: - self.error('Executing command {command} resulted in error: {error}'.format(command=self.command, + self.error('Executing command {command} resulted in error: {error}'.format(command=command or self.command, error=error)) return None data = list() @@ -35,7 +36,7 @@ class ExecutableService(SimpleService): except TypeError: continue - return data or None + return data def check(self): """ diff --git a/python.d/python_modules/bases/FrameworkServices/LogService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/LogService.py index 45daa2446..5acfd73f8 100644 --- a/python.d/python_modules/bases/FrameworkServices/LogService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/LogService.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # Description: # Author: Pawel Krupa (paulfantom) +# Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later from glob import glob import os diff --git a/python.d/python_modules/bases/FrameworkServices/MySQLService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py index 3acc5b109..53807e2c4 100644 --- a/python.d/python_modules/bases/FrameworkServices/MySQLService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Description: # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later from sys import exc_info diff --git a/python.d/python_modules/bases/FrameworkServices/SimpleService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py index 177332c1f..dd53fbc14 100644 --- a/python.d/python_modules/bases/FrameworkServices/SimpleService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py @@ -2,13 +2,12 @@ # Description: # Author: Pawel Krupa (paulfantom) # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later from threading import Thread +from time import sleep -try: - from time import sleep, monotonic as time -except ImportError: - from time import sleep, time +from third_party.monotonic import monotonic from bases.charts import Charts, ChartError, create_runtime_chart from bases.collection import OldVersionCompatibility, safe_print @@ -168,7 +167,7 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec 'retries: {retries}'.format(freq=job.FREQ, retries=job.RETRIES_MAX - job.RETRIES)) while True: - job.START_RUN = time() + job.START_RUN = monotonic() job.NEXT_RUN = job.START_RUN - (job.START_RUN % job.FREQ) + job.FREQ + job.PENALTY @@ -189,7 +188,7 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec if not self.manage_retries(): return else: - job.ELAPSED = int((time() - job.START_RUN) * 1e3) + job.ELAPSED = int((monotonic() - job.START_RUN) * 1e3) job.PREV_UPDATE = job.START_RUN job.RETRIES, job.PENALTY = 0, 0 safe_print(RUNTIME_CHART_UPDATE.format(job_name=self.name, @@ -253,7 +252,7 @@ class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, objec self.debug('sleeping for {sleep_time} to reach frequency of {freq} sec'.format(sleep_time=sleep_time, freq=job.FREQ + job.PENALTY)) sleep(sleep_time) - job.START_RUN = time() + job.START_RUN = monotonic() def get_data(self): return self._get_data() diff --git a/python.d/python_modules/bases/FrameworkServices/SocketService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py index 8d27ae660..e85455307 100644 --- a/python.d/python_modules/bases/FrameworkServices/SocketService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py @@ -1,9 +1,18 @@ # -*- coding: utf-8 -*- # Description: # Author: Pawel Krupa (paulfantom) +# Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later import socket +try: + import ssl +except ImportError: + _TLS_SUPPORT = False +else: + _TLS_SUPPORT = True + from bases.FrameworkServices.SimpleService import SimpleService @@ -16,6 +25,9 @@ class SocketService(SimpleService): self.unix_socket = None self.dgram_socket = False self.request = '' + self.tls = False + self.cert = None + self.key = None self.__socket_config = None self.__empty_request = "".encode() SimpleService.__init__(self, configuration=configuration, name=name) @@ -26,7 +38,7 @@ class SocketService(SimpleService): message=message)) else: if self.__socket_config is not None: - af, sock_type, proto, canon_name, sa = self.__socket_config + _, _, _, _, sa = self.__socket_config self.error('socket to "{address}" port {port}: {message}'.format(address=sa[0], port=sa[1], message=message)) @@ -44,7 +56,7 @@ class SocketService(SimpleService): self.error("Cannot create socket to 'None':") return False - af, sock_type, proto, canon_name, sa = res + af, sock_type, proto, _, sa = res try: self.debug('Creating socket to "{address}", port {port}'.format(address=sa[0], port=sa[1])) self._sock = socket.socket(af, sock_type, proto) @@ -56,10 +68,24 @@ class SocketService(SimpleService): self.__socket_config = None return False + if self.tls: + try: + self.debug('Encapsulating socket with TLS') + self._sock = ssl.wrap_socket(self._sock, + keyfile=self.key, + certfile=self.cert, + server_side=False, + cert_reqs=ssl.CERT_NONE) + except (socket.error, ssl.SSLError) as error: + self.error('Failed to wrap socket.') + self._disconnect() + self.__socket_config = None + return False + try: self.debug('connecting socket to "{address}", port {port}'.format(address=sa[0], port=sa[1])) self._sock.connect(sa) - except socket.error as error: + except (socket.error, ssl.SSLError) as error: self.error('Failed to connect to "{address}", port {port}, error: {error}'.format(address=sa[0], port=sa[1], error=error)) @@ -147,7 +173,7 @@ class SocketService(SimpleService): pass self._sock = None - def _send(self): + def _send(self, request=None): """ Send request. :return: boolean @@ -155,8 +181,8 @@ class SocketService(SimpleService): # Send request if it is needed if self.request != self.__empty_request: try: - self.debug('sending request: {0}'.format(self.request)) - self._sock.send(self.request) + self.debug('sending request: {0}'.format(request or self.request)) + self._sock.send(request or self.request) except Exception as error: self._socket_error('error sending request: {0}'.format(error)) self._disconnect() @@ -197,7 +223,7 @@ class SocketService(SimpleService): self.debug('final response: {0}'.format(data)) return data - def _get_raw_data(self, raw=False): + def _get_raw_data(self, raw=False, request=None): """ Get raw data with low-level "socket" module. :param raw: set `True` to return bytes @@ -211,7 +237,7 @@ class SocketService(SimpleService): return None # Send request if it is needed - if not self._send(): + if not self._send(request): return None data = self._receive(raw) @@ -249,6 +275,28 @@ class SocketService(SimpleService): except (KeyError, TypeError): self.debug('No port specified. Using: "{0}"'.format(self.port)) + self.tls = bool(self.configuration.get('tls', self.tls)) + if self.tls and not _TLS_SUPPORT: + self.warning('TLS requested but no TLS module found, disabling TLS support.') + self.tls = False + if _TLS_SUPPORT and not self.tls: + self.debug('No TLS preference specified, not using TLS.') + + if self.tls and _TLS_SUPPORT: + self.key = self.configuration.get('tls_key_file') + self.cert = self.configuration.get('tls_cert_file') + if not self.cert: + # If there's not a valid certificate, clear the key too. + self.debug('No valid TLS client certificate configuration found.') + self.key = None + self.cert = None + elif not self.key: + # If a key isn't listed, the config may still be + # valid, because there may be a key attached to the + # certificate. + self.info('No TLS client key specified, assuming it\'s attached to the certificate.') + self.key = None + try: self.request = str(self.configuration['request']) except (KeyError, TypeError): diff --git a/python.d/python_modules/bases/FrameworkServices/UrlService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py index bb340ba3b..856f38851 100644 --- a/python.d/python_modules/bases/FrameworkServices/UrlService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py @@ -2,6 +2,7 @@ # Description: # Author: Pawel Krupa (paulfantom) # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later import urllib3 @@ -22,8 +23,13 @@ class UrlService(SimpleService): self.proxy_user = self.configuration.get('proxy_user') self.proxy_password = self.configuration.get('proxy_pass') self.proxy_url = self.configuration.get('proxy_url') + self.method = self.configuration.get('method', 'GET') self.header = self.configuration.get('header') self.request_timeout = self.configuration.get('timeout', 1) + self.tls_verify = self.configuration.get('tls_verify') + self.tls_ca_file = self.configuration.get('tls_ca_file') + self.tls_key_file = self.configuration.get('tls_key_file') + self.tls_cert_file = self.configuration.get('tls_cert_file') self._manager = None def __make_headers(self, **header_kw): @@ -60,9 +66,21 @@ class UrlService(SimpleService): else: manager = urllib3.PoolManager params = dict(headers=header) + tls_cert_file = self.tls_cert_file + if tls_cert_file: + params['cert_file'] = tls_cert_file + # NOTE: key_file is useless without cert_file, but + # cert_file may include the key as well. + tls_key_file = self.tls_key_file + if tls_key_file: + params['key_file'] = tls_key_file + tls_ca_file = self.tls_ca_file + if tls_ca_file: + params['ca_certs'] = tls_ca_file try: url = header_kw.get('url') or self.url - if url.startswith('https'): + if url.startswith('https') and not self.tls_verify and not tls_ca_file: + params['ca_certs'] = None return manager(assert_hostname=False, cert_reqs='CERT_NONE', **params) return manager(**params) except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error: @@ -77,13 +95,13 @@ class UrlService(SimpleService): try: status, data = self._get_raw_data_with_status(url, manager) except (urllib3.exceptions.HTTPError, TypeError, AttributeError) as error: - self.error('Url: {url}. Error: {error}'.format(url=url, error=error)) + self.error('Url: {url}. Error: {error}'.format(url=url or self.url, error=error)) return None if status == 200: - return data.decode() + return data else: - self.debug('Url: {url}. Http response status code: {code}'.format(url=url, code=status)) + self.debug('Url: {url}. Http response status code: {code}'.format(url=url or self.url, code=status)) return None def _get_raw_data_with_status(self, url=None, manager=None, retries=1, redirect=True): @@ -93,13 +111,15 @@ class UrlService(SimpleService): """ url = url or self.url manager = manager or self._manager - response = manager.request(method='GET', + response = manager.request(method=self.method, url=url, timeout=self.request_timeout, retries=retries, headers=manager.headers, redirect=redirect) - return response.status, response.data + if isinstance(response.data, str): + return response.status, response.data + return response.status, response.data.decode() def check(self): """ diff --git a/python.d/python_modules/bases/charts.py b/collectors/python.d.plugin/python_modules/bases/charts.py index 5394fbf64..2963739ec 100644 --- a/python.d/python_modules/bases/charts.py +++ b/collectors/python.d.plugin/python_modules/bases/charts.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # Description: # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later from bases.collection import safe_print -CHART_PARAMS = ['type', 'id', 'name', 'title', 'units', 'family', 'context', 'chart_type'] +CHART_PARAMS = ['type', 'id', 'name', 'title', 'units', 'family', 'context', 'chart_type', 'hidden'] DIMENSION_PARAMS = ['id', 'name', 'algorithm', 'multiplier', 'divisor', 'hidden'] VARIABLE_PARAMS = ['id', 'value'] @@ -13,9 +14,9 @@ DIMENSION_ALGORITHMS = ['absolute', 'incremental', 'percentage-of-absolute-row', CHART_BEGIN = 'BEGIN {type}.{id} {since_last}\n' CHART_CREATE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \ - "{chart_type} {priority} {update_every} '' 'python.d.plugin' '{module_name}'\n" + "{chart_type} {priority} {update_every} '{hidden}' 'python.d.plugin' '{module_name}'\n" CHART_OBSOLETE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \ - "{chart_type} {priority} {update_every} 'obsolete'\n" + "{chart_type} {priority} {update_every} '{hidden} obsolete'\n" DIMENSION_CREATE = "DIMENSION '{id}' '{name}' {algorithm} {multiplier} {divisor} '{hidden}'\n" @@ -151,6 +152,8 @@ class Chart: id=self.params['id']) if self.params.get('chart_type') not in CHART_TYPES: self.params['chart_type'] = 'absolute' + hidden = str(self.params.get('hidden', '')) + self.params['hidden'] = 'hidden' if hidden == 'hidden' else '' self.dimensions = list() self.variables = set() @@ -304,6 +307,12 @@ class Dimension: return self.id == other return self.id == other.id + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(repr(self)) + def create(self): return DIMENSION_CREATE.format(**self.params) @@ -360,6 +369,9 @@ class ChartVariable: return self.id == other.id return False + def __ne__(self, other): + return not self == other + def __hash__(self): return hash(repr(self)) diff --git a/python.d/python_modules/bases/collection.py b/collectors/python.d.plugin/python_modules/bases/collection.py index e03b4f58e..479a3b610 100644 --- a/python.d/python_modules/bases/collection.py +++ b/collectors/python.d.plugin/python_modules/bases/collection.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Description: # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later import os diff --git a/python.d/python_modules/bases/loaders.py b/collectors/python.d.plugin/python_modules/bases/loaders.py index d18b9dcd0..9eb268ce7 100644 --- a/python.d/python_modules/bases/loaders.py +++ b/collectors/python.d.plugin/python_modules/bases/loaders.py @@ -1,18 +1,27 @@ # -*- coding: utf-8 -*- # Description: # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later import types + from sys import version_info PY_VERSION = version_info[:2] +try: + if PY_VERSION > (3, 1): + from pyyaml3 import SafeLoader as YamlSafeLoader + else: + from pyyaml2 import SafeLoader as YamlSafeLoader +except ImportError: + from yaml import SafeLoader as YamlSafeLoader + + if PY_VERSION > (3, 1): - from pyyaml3 import SafeLoader as YamlSafeLoader from importlib.machinery import SourceFileLoader DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' else: - from pyyaml2 import SafeLoader as YamlSafeLoader from imp import load_source as SourceFileLoader DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map' @@ -26,6 +35,14 @@ def dict_constructor(loader, node): return OrderedDict(loader.construct_pairs(node)) +def safe_load(stream): + loader = YamlSafeLoader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() + + YamlSafeLoader.add_constructor(DEFAULT_MAPPING_TAG, dict_constructor) diff --git a/python.d/python_modules/bases/loggers.py b/collectors/python.d.plugin/python_modules/bases/loggers.py index fc40b83d3..39be77a79 100644 --- a/python.d/python_modules/bases/loggers.py +++ b/collectors/python.d.plugin/python_modules/bases/loggers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Description: # Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0-or-later import logging import traceback |