summaryrefslogtreecommitdiffstats
path: root/python.d/python_modules
diff options
context:
space:
mode:
authorLennart Weller <lhw@ring0.de>2017-07-27 09:55:47 +0000
committerLennart Weller <lhw@ring0.de>2017-07-27 09:55:47 +0000
commita133c9c3b637b1dbe7b5b053f7e2572c1950cead (patch)
tree2207939a88e96bca329457f40a9d9d18ab659dc1 /python.d/python_modules
parentNew upstream version 1.6.0+dfsg (diff)
downloadnetdata-a133c9c3b637b1dbe7b5b053f7e2572c1950cead.tar.xz
netdata-a133c9c3b637b1dbe7b5b053f7e2572c1950cead.zip
New upstream version 1.7.0+dfsgupstream/1.7.0+dfsg
Diffstat (limited to 'python.d/python_modules')
-rw-r--r--python.d/python_modules/base.py211
1 files changed, 141 insertions, 70 deletions
diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py
index 859300eca..a643cc6a0 100644
--- a/python.d/python_modules/base.py
+++ b/python.d/python_modules/base.py
@@ -20,23 +20,20 @@
import time
import os
import socket
-import select
import threading
-import msg
import ssl
from subprocess import Popen, PIPE
from sys import exc_info
-
+from glob import glob
+import re
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
-
try:
import urllib.request as urllib2
except ImportError:
import urllib2
-
try:
import MySQLdb
PYMYSQL = True
@@ -46,6 +43,7 @@ except ImportError:
PYMYSQL = True
except ImportError:
PYMYSQL = False
+import msg
try:
PATH = os.getenv('PATH').split(':')
@@ -175,14 +173,15 @@ class SimpleService(threading.Thread):
# it is important to do this in a loop
# sleep() is interruptable
while now < next:
- self.debug("sleeping for", str(next - now), "secs to reach frequency of", str(step), "secs, now:", str(now), " next:", str(next), " penalty:", str(penalty))
+ self.debug("sleeping for", str(next - now), "secs to reach frequency of",
+ str(step), "secs, now:", str(now), " next:", str(next), " penalty:", str(penalty))
time.sleep(next - now)
now = float(time.time())
# do the job
try:
status = self._run_once()
- except Exception as e:
+ except Exception:
status = False
if status:
@@ -202,10 +201,12 @@ class SimpleService(threading.Thread):
penalty = 600
self.retries_left = self.retries
- self.alert("failed to collect data for " + str(self.retries) + " times - increasing penalty to " + str(penalty) + " sec and trying again")
+ self.alert("failed to collect data for " + str(self.retries) +
+ " times - increasing penalty to " + str(penalty) + " sec and trying again")
else:
- self.error("failed to collect data - " + str(self.retries_left) + " retries left - penalty: " + str(penalty) + " sec")
+ self.error("failed to collect data - " + str(self.retries_left)
+ + " retries left - penalty: " + str(penalty) + " sec")
# --- CHART ---
@@ -460,11 +461,42 @@ class SimpleService(threading.Thread):
return next(('/'.join([p, binary]) for p in PATH
if os.path.isfile('/'.join([p, binary]))
and os.access('/'.join([p, binary]), os.X_OK)))
- else:
- return None
+ return None
except StopIteration:
return None
+ def _add_new_dimension(self, dimension_id, chart_name, dimension=None, algorithm='incremental',
+ multiplier=1, divisor=1, priority=65000):
+ """
+ :param dimension_id:
+ :param chart_name:
+ :param dimension:
+ :param algorithm:
+ :param multiplier:
+ :param divisor:
+ :param priority:
+ :return:
+ """
+ if not all([dimension_id not in self._dimensions,
+ chart_name in self.order,
+ chart_name in self.definitions]):
+ return
+ self._dimensions.append(dimension_id)
+ dimension_list = list(map(str, [dimension_id,
+ dimension if dimension else dimension_id,
+ algorithm,
+ multiplier,
+ divisor]))
+ self.definitions[chart_name]['lines'].append(dimension_list)
+ add_to_name = self.override_name or self.name
+ job_name = ('_'.join([self.__module__, re.sub('\s+', '_', add_to_name)])
+ if add_to_name != 'None' else self.__module__)
+ chart = 'CHART {0}.{1} '.format(job_name, chart_name)
+ options = '"" "{0}" {1} "{2}" {3} {4} '.format(*self.definitions[chart_name]['options'][1:6])
+ other = '{0} {1}\n'.format(priority, self.update_every)
+ new_dimension = "DIMENSION {0}\n".format(' '.join(dimension_list))
+ print(chart + options + other + new_dimension)
+
class UrlService(SimpleService):
def __init__(self, configuration=None, name=None):
@@ -473,47 +505,73 @@ class UrlService(SimpleService):
self.user = self.configuration.get('user')
self.password = self.configuration.get('pass')
self.ss_cert = self.configuration.get('ss_cert')
+ self.proxy = self.configuration.get('proxy')
- def __add_openers(self):
- def self_signed_cert(ss_cert):
- if ss_cert:
- try:
- ctx = ssl.create_default_context()
- ctx.check_hostname = False
- ctx.verify_mode = ssl.CERT_NONE
- return urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
- except AttributeError:
- return None
- else:
- return None
+ def __add_openers(self, user=None, password=None, ss_cert=None, proxy=None, url=None):
+ user = user or self.user
+ password = password or self.password
+ ss_cert = ss_cert or self.ss_cert
+ proxy = proxy or self.proxy
- self.opener = self_signed_cert(self.ss_cert) or urllib2.build_opener()
+ handlers = list()
- # HTTP Basic Auth
- if self.user and self.password:
- url_parse = urlparse(self.url)
+ # HTTP Basic Auth handler
+ if all([user, password, isinstance(user, str), isinstance(password, str)]):
+ url = url or self.url
+ url_parse = urlparse(url)
top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
- passman.add_password(None, top_level_url, self.user, self.password)
- self.opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
+ passman.add_password(None, top_level_url, user, password)
+ handlers.append(urllib2.HTTPBasicAuthHandler(passman))
self.debug("Enabling HTTP basic auth")
- def _get_raw_data(self, custom_url=None):
+ # HTTPS handler
+ # Self-signed certificate ignore
+ if ss_cert:
+ try:
+ ctx = ssl.create_default_context()
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ except AttributeError:
+ self.error('HTTPS self-signed certificate ignore not enabled')
+ else:
+ handlers.append(urllib2.HTTPSHandler(context=ctx))
+ self.debug("Enabling HTTP self-signed certificate ignore")
+
+ # PROXY handler
+ if proxy and isinstance(proxy, str) and not ss_cert:
+ handlers.append(urllib2.ProxyHandler(dict(http=proxy)))
+ self.debug("Enabling HTTP proxy handler (%s)" % proxy)
+
+ opener = urllib2.build_opener(*handlers)
+ return opener
+
+ def _build_opener(self, **kwargs):
+ try:
+ return self.__add_openers(**kwargs)
+ except TypeError as error:
+ self.error('build_opener() error:', str(error))
+ return None
+
+ def _get_raw_data(self, url=None, opener=None):
"""
Get raw data from http request
:return: str
"""
- raw_data = None
- f = None
+ data = None
try:
- f = self.opener.open(custom_url or self.url, timeout=self.update_every * 2)
- raw_data = f.read().decode('utf-8', 'ignore')
+ opener = opener or self.opener
+ data = opener.open(url or self.url, timeout=self.update_every * 2)
+ raw_data = data.read().decode('utf-8', 'ignore')
+ except urllib2.URLError as error:
+ self.error('Url: %s. Error: %s' % (url or self.url, str(error)))
+ return None
except Exception as error:
- self.error('Url: %s. Error: %s' %(custom_url or self.url, str(error)))
+ self.error(str(error))
return None
finally:
- if f is not None: f.close()
-
+ if data is not None:
+ data.close()
return raw_data or None
def check(self):
@@ -525,7 +583,7 @@ class UrlService(SimpleService):
self.error('URL is not defined or type is not <str>')
return False
- self.__add_openers()
+ self.opener = self.__add_openers()
try:
data = self._get_data()
@@ -781,57 +839,69 @@ class SocketService(SimpleService):
class LogService(SimpleService):
def __init__(self, configuration=None, name=None):
- self.log_path = ""
- self._last_position = 0
- # self._log_reader = None
SimpleService.__init__(self, configuration=configuration, name=name)
+ self.log_path = self.configuration.get('path')
+ self.__glob_path = self.log_path
+ self._last_position = 0
self.retries = 100000 # basically always retry
+ self.__re_find = dict(current=0, run=0, maximum=60)
def _get_raw_data(self):
"""
Get log lines since last poll
:return: list
"""
- lines = []
+ lines = list()
try:
- if os.path.getsize(self.log_path) < self._last_position:
+ if self.__re_find['current'] == self.__re_find['run']:
+ self._find_recent_log_file()
+ size = os.path.getsize(self.log_path)
+ if size == self._last_position:
+ self.__re_find['current'] += 1
+ return list() # return empty list if nothing has changed
+ elif size < self._last_position:
self._last_position = 0 # read from beginning if file has shrunk
- elif os.path.getsize(self.log_path) == self._last_position:
- self.debug("Log file hasn't changed. No new data.")
- return [] # return empty list if nothing has changed
- with open(self.log_path, "r") as fp:
+
+ with open(self.log_path) as fp:
fp.seek(self._last_position)
- for i, line in enumerate(fp):
+ for line in fp:
lines.append(line)
self._last_position = fp.tell()
- except Exception as e:
- self.error(str(e))
+ self.__re_find['current'] = 0
+ except (OSError, IOError) as error:
+ self.__re_find['current'] += 1
+ self.error(str(error))
- if len(lines) != 0:
- return lines
- else:
- self.error("No data collected.")
- return None
+ return lines or None
+
+ def _find_recent_log_file(self):
+ """
+ :return:
+ """
+ self.__re_find['run'] = self.__re_find['maximum']
+ self.__re_find['current'] = 0
+ self.__glob_path = self.__glob_path or self.log_path # workaround for modules w/o config files
+ path_list = glob(self.__glob_path)
+ if path_list:
+ self.log_path = max(path_list)
+ return True
+ return False
def check(self):
"""
Parse basic configuration and check if log file exists
:return: boolean
"""
- if self.name is not None or self.name != str(None):
- self.name = ""
- else:
- self.name = str(self.name)
- try:
- self.log_path = str(self.configuration['path'])
- except (KeyError, TypeError):
- self.info("No path to log specified. Using: '" + self.log_path + "'")
+ if not self.log_path:
+ self.error("No path to log specified")
+ return None
- if os.access(self.log_path, os.R_OK):
+ if all([self._find_recent_log_file(),
+ os.access(self.log_path, os.R_OK),
+ os.path.isfile(self.log_path)]):
return True
- else:
- self.error("Cannot access file: '" + self.log_path + "'")
- return False
+ self.error("Cannot access %s" % self.log_path)
+ return False
def create(self):
# set cursor at last byte of log file
@@ -847,7 +917,7 @@ class ExecutableService(SimpleService):
SimpleService.__init__(self, configuration=configuration, name=name)
self.command = None
- def _get_raw_data(self):
+ def _get_raw_data(self, stderr=False):
"""
Get raw data from executed command
:return: <list>
@@ -855,10 +925,11 @@ class ExecutableService(SimpleService):
try:
p = Popen(self.command, stdout=PIPE, stderr=PIPE)
except Exception as error:
- self.error("Executing command", self.command, "resulted in error:", str(error))
+ self.error("Executing command", " ".join(self.command), "resulted in error:", str(error))
return None
data = list()
- for line in p.stdout.readlines():
+ std = p.stderr if stderr else p.stdout
+ for line in std.readlines():
data.append(line.decode())
return data or None