diff options
Diffstat (limited to 'collectors/python.d.plugin/python_modules/bases')
8 files changed, 91 insertions, 31 deletions
diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py index f63cb7c2f..dea50eea0 100644 --- a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/ExecutableService.py @@ -22,12 +22,14 @@ class ExecutableService(SimpleService): Get raw data from executed command :return: <list> """ + command = command or self.command + self.debug("Executing command '{0}'".format(' '.join(command))) try: - p = Popen(command if command else self.command, stdout=PIPE, stderr=PIPE) + p = Popen(command, stdout=PIPE, stderr=PIPE) except Exception as error: - self.error('Executing command {command} resulted in error: {error}'.format(command=command or self.command, - error=error)) + self.error('Executing command {0} resulted in error: {1}'.format(command, error)) return None + data = list() std = p.stderr if stderr else p.stdout for line in std: diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py index 354d09ad8..7f5c7d221 100644 --- a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/MySQLService.py @@ -51,11 +51,7 @@ class MySQLService(SimpleService): properties['host'] = conf['host'] properties['port'] = int(conf.get('port', 3306)) elif conf.get('my.cnf'): - if MySQLdb.__name__ == 'pymysql': - # TODO: this is probablt wrong, it depends on version - self.error('"my.cnf" parsing is not working for pymysql') - else: - properties['read_default_file'] = conf['my.cnf'] + properties['read_default_file'] = conf['my.cnf'] if conf.get('ssl'): properties['ssl'] = conf['ssl'] diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py index 4dfd226b0..c304ccec2 100644 --- a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SimpleService.py @@ -55,11 +55,18 @@ class RuntimeCounters: self.penalty = round(min(self.retries * self.update_every / 2, MAX_PENALTY)) +def clean_module_name(name): + if name.startswith('pythond_'): + return name[8:] + return name + + class SimpleService(PythonDLimitedLogger, object): """ Prototype of Service class. Implemented basic functionality to run jobs by `python.d.plugin` """ + def __init__(self, configuration, name=''): """ :param configuration: <dict> @@ -70,7 +77,7 @@ class SimpleService(PythonDLimitedLogger, object): self.order = list() self.definitions = dict() - self.module_name = self.__module__ + self.module_name = clean_module_name(self.__module__) self.job_name = configuration.pop('job_name') self.override_name = configuration.pop('override_name') self.fake_name = None diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py index 337bf57d8..bef3792da 100644 --- a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/SocketService.py @@ -247,7 +247,7 @@ class SocketService(SimpleService): if self._check_raw_data(data): break - self.debug('final response: {0}'.format(data)) + self.debug(u'final response: {0}'.format(data)) return data def _get_raw_data(self, raw=False, request=None): diff --git a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py index cfc7899e5..1faf036a4 100644 --- a/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py +++ b/collectors/python.d.plugin/python_modules/bases/FrameworkServices/UrlService.py @@ -47,6 +47,7 @@ class UrlService(SimpleService): self.proxy_url = self.configuration.get('proxy_url') self.method = self.configuration.get('method', 'GET') self.header = self.configuration.get('header') + self.body = self.configuration.get('body') self.request_timeout = self.configuration.get('timeout', 1) self.respect_retry_after_header = self.configuration.get('respect_retry_after_header') self.tls_verify = self.configuration.get('tls_verify') @@ -119,15 +120,17 @@ class UrlService(SimpleService): :return: str """ try: - status, data = self._get_raw_data_with_status(url, manager, **kwargs) + response = self._do_request(url, manager, **kwargs) except Exception as error: self.error('Url: {url}. Error: {error}'.format(url=url or self.url, error=error)) return None - if status == 200: - return data + if response.status == 200: + if isinstance(response.data, str): + return response.data + return response.data.decode(errors='ignore') else: - self.debug('Url: {url}. Http response status code: {code}'.format(url=url or self.url, code=status)) + self.debug('Url: {url}. Http response status code: {code}'.format(url=url or self.url, code=response.status)) return None def _get_raw_data_with_status(self, url=None, manager=None, retries=1, redirect=True, **kwargs): @@ -135,12 +138,26 @@ class UrlService(SimpleService): Get status and response body content from http request. Does not catch exceptions :return: int, str """ + response = self._do_request(url, manager, retries, redirect, **kwargs) + + if isinstance(response.data, str): + return response.status, response.data + return response.status, response.data.decode(errors='ignore') + + def _do_request(self, url=None, manager=None, retries=1, redirect=True, **kwargs): + """ + Get response from http request. Does not catch exceptions + :return: HTTPResponse + """ url = url or self.url manager = manager or self._manager retry = urllib3.Retry(retries) if hasattr(retry, 'respect_retry_after_header'): retry.respect_retry_after_header = bool(self.respect_retry_after_header) + if self.body: + kwargs['body'] = self.body + response = manager.request( method=self.method, url=url, @@ -150,9 +167,7 @@ class UrlService(SimpleService): redirect=redirect, **kwargs ) - if isinstance(response.data, str): - return response.status, response.data - return response.status, response.data.decode() + return response def check(self): """ diff --git a/collectors/python.d.plugin/python_modules/bases/charts.py b/collectors/python.d.plugin/python_modules/bases/charts.py index 6e78ed6e7..93be43d14 100644 --- a/collectors/python.d.plugin/python_modules/bases/charts.py +++ b/collectors/python.d.plugin/python_modules/bases/charts.py @@ -16,8 +16,7 @@ CHART_BEGIN = 'BEGIN {type}.{id} {since_last}\n' CHART_CREATE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \ "{chart_type} {priority} {update_every} '{hidden}' 'python.d.plugin' '{module_name}'\n" CHART_OBSOLETE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \ - "{chart_type} {priority} {update_every} '{hidden} obsolete'\n" - + "{chart_type} {priority} {update_every} '{hidden} obsolete'\n" DIMENSION_CREATE = "DIMENSION '{id}' '{name}' {algorithm} {multiplier} {divisor} '{hidden} {obsolete}'\n" DIMENSION_SET = "SET '{id}' = {value}\n" @@ -40,13 +39,17 @@ def create_runtime_chart(func): :param func: class method :return: """ + def wrapper(*args, **kwargs): self = args[0] + chart = RUNTIME_CHART_CREATE.format( + job_name=self.name, + update_every=self._runtime_counters.update_every, + ) + safe_print(chart) ok = func(*args, **kwargs) - if ok: - safe_print(RUNTIME_CHART_CREATE.format(job_name=self.name, - update_every=self._runtime_counters.update_every)) return ok + return wrapper @@ -72,6 +75,7 @@ class Charts: All charts stored in a dict. Chart is a instance of Chart class. Charts adding must be done using Charts.add_chart() method only""" + def __init__(self, job_name, priority, cleanup, get_update_every, module_name): """ :param job_name: <bound method> @@ -138,6 +142,7 @@ class Charts: class Chart: """Represent a chart""" + def __init__(self, params): """ :param params: <list> @@ -281,6 +286,7 @@ class Chart: class Dimension: """Represent a dimension""" + def __init__(self, params): """ :param params: <list> @@ -346,6 +352,7 @@ class Dimension: class ChartVariable: """Represent a chart variable""" + def __init__(self, params): """ :param params: <list> diff --git a/collectors/python.d.plugin/python_modules/bases/collection.py b/collectors/python.d.plugin/python_modules/bases/collection.py index 4c25aafd5..93bf8cf05 100644 --- a/collectors/python.d.plugin/python_modules/bases/collection.py +++ b/collectors/python.d.plugin/python_modules/bases/collection.py @@ -5,6 +5,8 @@ import os +from threading import Lock + PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':') CHART_BEGIN = 'BEGIN {0} {1}\n' @@ -12,6 +14,8 @@ CHART_CREATE = "CHART {0} '{1}' '{2}' '{3}' '{4}' '{5}' {6} {7} {8}\n" DIMENSION_CREATE = "DIMENSION '{0}' '{1}' {2} {3} {4} '{5}'\n" DIMENSION_SET = "SET '{0}' = {1}\n" +print_lock = Lock() + def setdefault_values(config, base_dict): for key, value in base_dict.items(): @@ -23,10 +27,11 @@ def run_and_exit(func): def wrapper(*args, **kwargs): func(*args, **kwargs) exit(1) + return wrapper -def on_try_except_finally(on_except=(None, ), on_finally=(None, )): +def on_try_except_finally(on_except=(None,), on_finally=(None,)): except_func = on_except[0] finally_func = on_finally[0] @@ -40,7 +45,9 @@ def on_try_except_finally(on_except=(None, ), on_finally=(None, )): finally: if finally_func: finally_func(*on_finally[1:]) + return wrapper + return decorator @@ -49,6 +56,7 @@ def static_vars(**kwargs): for k in kwargs: setattr(func, k, kwargs[k]) return func + return decorate @@ -58,7 +66,9 @@ def safe_print(*msg): :param msg: :return: """ + print_lock.acquire() print(''.join(msg)) + print_lock.release() def find_binary(binary): @@ -67,7 +77,7 @@ def find_binary(binary): :return: """ for directory in PATH: - binary_name = '/'.join([directory, binary]) + binary_name = os.path.join(directory, binary) if os.path.isfile(binary_name) and os.access(binary_name, os.X_OK): return binary_name return None @@ -82,3 +92,26 @@ def read_last_line(f): break result = opened.readline() return result.decode() + + +def unicode_str(arg): + """Return the argument as a unicode string. + + The `unicode` function has been removed from Python3 and `str` takes its + place. This function is a helper which will try using Python 2's `unicode` + and if it doesn't exist, assume we're using Python 3 and use `str`. + + :param arg: + :return: <str> + """ + # TODO: fix + try: + # https://github.com/netdata/netdata/issues/7613 + if isinstance(arg, unicode): + return arg + return unicode(arg, errors='ignore') + # https://github.com/netdata/netdata/issues/7642 + except TypeError: + return unicode(arg) + except NameError: + return str(arg) diff --git a/collectors/python.d.plugin/python_modules/bases/loggers.py b/collectors/python.d.plugin/python_modules/bases/loggers.py index 9bf2e086b..47f196a6d 100644 --- a/collectors/python.d.plugin/python_modules/bases/loggers.py +++ b/collectors/python.d.plugin/python_modules/bases/loggers.py @@ -13,7 +13,7 @@ try: except ImportError: from time import time -from bases.collection import on_try_except_finally +from bases.collection import on_try_except_finally, unicode_str LOGGING_LEVELS = {'CRITICAL': 50, @@ -121,23 +121,23 @@ class BaseLogger(object): self.logger.setLevel(LOGGING_LEVELS[level]) def debug(self, *msg, **kwargs): - self.logger.debug(' '.join(map(str, msg)), **kwargs) + self.logger.debug(' '.join(map(unicode_str, msg)), **kwargs) def info(self, *msg, **kwargs): - self.logger.info(' '.join(map(str, msg)), **kwargs) + self.logger.info(' '.join(map(unicode_str, msg)), **kwargs) def warning(self, *msg, **kwargs): - self.logger.warning(' '.join(map(str, msg)), **kwargs) + self.logger.warning(' '.join(map(unicode_str, msg)), **kwargs) def error(self, *msg, **kwargs): - self.logger.error(' '.join(map(str, msg)), **kwargs) + self.logger.error(' '.join(map(unicode_str, msg)), **kwargs) def alert(self, *msg, **kwargs): - self.logger.critical(' '.join(map(str, msg)), **kwargs) + self.logger.critical(' '.join(map(unicode_str, msg)), **kwargs) @on_try_except_finally(on_finally=(exit, 1)) def fatal(self, *msg, **kwargs): - self.logger.critical(' '.join(map(str, msg)), **kwargs) + self.logger.critical(' '.join(map(unicode_str, msg)), **kwargs) class PythonDLogger(object): |