diff options
Diffstat (limited to 'python.d/python_modules/base.py')
| -rw-r--r-- | python.d/python_modules/base.py | 171 |
1 files changed, 83 insertions, 88 deletions
diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py index a643cc6a0..1d5417ec2 100644 --- a/python.d/python_modules/base.py +++ b/python.d/python_modules/base.py @@ -17,38 +17,36 @@ # # using ".encode()" in one thread can block other threads as well (only in python2) -import time import os +import re import socket +import time import threading -import ssl + +import urllib3 + +from glob import glob from subprocess import Popen, PIPE from sys import exc_info -from glob import glob -import re -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse -try: - import urllib.request as urllib2 -except ImportError: - import urllib2 + try: import MySQLdb - PYMYSQL = True + PY_MYSQL = True except ImportError: try: import pymysql as MySQLdb - PYMYSQL = True + PY_MYSQL = True except ImportError: - PYMYSQL = False + PY_MYSQL = False + import msg + +PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':') try: - PATH = os.getenv('PATH').split(':') + urllib3.disable_warnings() except AttributeError: - PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':') + msg.error('urllib3: warnings were not disabled') # class BaseService(threading.Thread): @@ -504,75 +502,68 @@ class UrlService(SimpleService): self.url = self.configuration.get('url') self.user = self.configuration.get('user') self.password = self.configuration.get('pass') - self.ss_cert = self.configuration.get('ss_cert') - self.proxy = self.configuration.get('proxy') - - def __add_openers(self, user=None, password=None, ss_cert=None, proxy=None, url=None): - user = user or self.user - password = password or self.password - ss_cert = ss_cert or self.ss_cert - proxy = proxy or self.proxy - - handlers = list() - - # HTTP Basic Auth handler - if all([user, password, isinstance(user, str), isinstance(password, str)]): - url = url or self.url - url_parse = urlparse(url) - top_level_url = '://'.join([url_parse.scheme, url_parse.netloc]) - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, top_level_url, user, password) - handlers.append(urllib2.HTTPBasicAuthHandler(passman)) - self.debug("Enabling HTTP basic auth") - - # HTTPS handler - # Self-signed certificate ignore - if ss_cert: - try: - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - except AttributeError: - self.error('HTTPS self-signed certificate ignore not enabled') - else: - handlers.append(urllib2.HTTPSHandler(context=ctx)) - self.debug("Enabling HTTP self-signed certificate ignore") - - # PROXY handler - if proxy and isinstance(proxy, str) and not ss_cert: - handlers.append(urllib2.ProxyHandler(dict(http=proxy))) - self.debug("Enabling HTTP proxy handler (%s)" % proxy) - - opener = urllib2.build_opener(*handlers) - return opener - - def _build_opener(self, **kwargs): + self.proxy_user = self.configuration.get('proxy_user') + self.proxy_password = self.configuration.get('proxy_pass') + self.proxy_url = self.configuration.get('proxy_url') + self._manager = None + + def __make_headers(self, **header_kw): + user = header_kw.get('user') or self.user + password = header_kw.get('pass') or self.password + proxy_user = header_kw.get('proxy_user') or self.proxy_user + proxy_password = header_kw.get('proxy_pass') or self.proxy_password + header_params = dict(keep_alive=True) + proxy_header_params = dict() + if user and password: + header_params['basic_auth'] = '{user}:{password}'.format(user=user, + password=password) + if proxy_user and proxy_password: + proxy_header_params['proxy_basic_auth'] = '{user}:{password}'.format(user=proxy_user, + password=proxy_password) try: - return self.__add_openers(**kwargs) + return urllib3.make_headers(**header_params), urllib3.make_headers(**proxy_header_params) except TypeError as error: - self.error('build_opener() error:', str(error)) + self.error('build_header() error: {error}'.format(error=error)) + return None, None + + def _build_manager(self, **header_kw): + header, proxy_header = self.__make_headers(**header_kw) + if header is None or proxy_header is None: + return None + proxy_url = header_kw.get('proxy_url') or self.proxy_url + if proxy_url: + manager = urllib3.ProxyManager + params = dict(proxy_url=proxy_url, headers=header, proxy_headers=proxy_header) + else: + manager = urllib3.PoolManager + params = dict(headers=header) + try: + return manager(**params) + except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error: + self.error('build_manager() error:', str(error)) return None - def _get_raw_data(self, url=None, opener=None): + def _get_raw_data(self, url=None, manager=None): """ Get raw data from http request :return: str """ - data = None try: - opener = opener or self.opener - data = opener.open(url or self.url, timeout=self.update_every * 2) - raw_data = data.read().decode('utf-8', 'ignore') - except urllib2.URLError as error: - self.error('Url: %s. Error: %s' % (url or self.url, str(error))) - return None - except Exception as error: - self.error(str(error)) + url = url or self.url + manager = manager or self._manager + # TODO: timeout, retries and method hardcoded.. + response = manager.request(method='GET', + url=url, + timeout=1, + retries=1, + headers=manager.headers) + except (urllib3.exceptions.HTTPError, TypeError, AttributeError) as error: + self.error('Url: {url}. Error: {error}'.format(url=url, error=error)) return None - finally: - if data is not None: - data.close() - return raw_data or None + if response.status == 200: + return response.data.decode() + self.debug('Url: {url}. Http response status code: {code}'.format(url=url, code=response.status)) + return None def check(self): """ @@ -583,20 +574,21 @@ class UrlService(SimpleService): self.error('URL is not defined or type is not <str>') return False - self.opener = self.__add_openers() + self._manager = self._build_manager() + if not self._manager: + return False try: data = self._get_data() except Exception as error: - self.error('_get_data() failed. Url: %s. Error: %s' % (self.url, error)) + self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error)) return False if isinstance(data, dict) and data: self._data_from_check = data return True - else: - self.error("_get_data() returned no data or type is not <dict>") - return False + self.error('_get_data() returned no data or type is not <dict>') + return False class SocketService(SimpleService): @@ -1007,17 +999,20 @@ class MySQLService(SimpleService): def check(self): def get_connection_properties(conf, extra_conf): properties = dict() - if 'user' in conf and conf['user']: + if conf.get('user'): properties['user'] = conf['user'] - if 'pass' in conf and conf['pass']: + if conf.get('pass'): properties['passwd'] = conf['pass'] - if 'socket' in conf and conf['socket']: + if conf.get('socket'): properties['unix_socket'] = conf['socket'] - elif 'host' in conf and conf['host']: + elif conf.get('host'): properties['host'] = conf['host'] properties['port'] = int(conf.get('port', 3306)) - elif 'my.cnf' in conf and conf['my.cnf']: - properties['read_default_file'] = conf['my.cnf'] + elif conf.get('my.cnf'): + if MySQLdb.__name__ == 'pymysql': + self.error('"my.cnf" parsing is not working for pymysql') + else: + properties['read_default_file'] = conf['my.cnf'] if isinstance(extra_conf, dict) and extra_conf: properties.update(extra_conf) @@ -1046,7 +1041,7 @@ class MySQLService(SimpleService): log_error('Unsupported "queries" format. Must be not empty <dict>') return None - if not PYMYSQL: + if not PY_MYSQL: self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin') return False |