summaryrefslogtreecommitdiffstats
path: root/python.d/python_modules/base.py
diff options
context:
space:
mode:
authorLennart Weller <lhw@ring0.de>2017-09-17 22:17:33 +0000
committerLennart Weller <lhw@ring0.de>2017-09-17 22:17:33 +0000
commit6aaf5ba7ed0980c14bdc554fc8839a2126455ed5 (patch)
tree6161925716661486e7f47c479668a9487b039d83 /python.d/python_modules/base.py
parentNew upstream version 1.7.0+dfsg (diff)
downloadnetdata-6aaf5ba7ed0980c14bdc554fc8839a2126455ed5.tar.xz
netdata-6aaf5ba7ed0980c14bdc554fc8839a2126455ed5.zip
New upstream version 1.8.0+dfsgupstream/1.8.0+dfsg
Diffstat (limited to 'python.d/python_modules/base.py')
-rw-r--r--python.d/python_modules/base.py171
1 files changed, 83 insertions, 88 deletions
diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py
index a643cc6a0..1d5417ec2 100644
--- a/python.d/python_modules/base.py
+++ b/python.d/python_modules/base.py
@@ -17,38 +17,36 @@
#
# using ".encode()" in one thread can block other threads as well (only in python2)
-import time
import os
+import re
import socket
+import time
import threading
-import ssl
+
+import urllib3
+
+from glob import glob
from subprocess import Popen, PIPE
from sys import exc_info
-from glob import glob
-import re
-try:
- from urlparse import urlparse
-except ImportError:
- from urllib.parse import urlparse
-try:
- import urllib.request as urllib2
-except ImportError:
- import urllib2
+
try:
import MySQLdb
- PYMYSQL = True
+ PY_MYSQL = True
except ImportError:
try:
import pymysql as MySQLdb
- PYMYSQL = True
+ PY_MYSQL = True
except ImportError:
- PYMYSQL = False
+ PY_MYSQL = False
+
import msg
+
+PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':')
try:
- PATH = os.getenv('PATH').split(':')
+ urllib3.disable_warnings()
except AttributeError:
- PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':')
+ msg.error('urllib3: warnings were not disabled')
# class BaseService(threading.Thread):
@@ -504,75 +502,68 @@ class UrlService(SimpleService):
self.url = self.configuration.get('url')
self.user = self.configuration.get('user')
self.password = self.configuration.get('pass')
- self.ss_cert = self.configuration.get('ss_cert')
- self.proxy = self.configuration.get('proxy')
-
- def __add_openers(self, user=None, password=None, ss_cert=None, proxy=None, url=None):
- user = user or self.user
- password = password or self.password
- ss_cert = ss_cert or self.ss_cert
- proxy = proxy or self.proxy
-
- handlers = list()
-
- # HTTP Basic Auth handler
- if all([user, password, isinstance(user, str), isinstance(password, str)]):
- url = url or self.url
- url_parse = urlparse(url)
- top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
- passman.add_password(None, top_level_url, user, password)
- handlers.append(urllib2.HTTPBasicAuthHandler(passman))
- self.debug("Enabling HTTP basic auth")
-
- # HTTPS handler
- # Self-signed certificate ignore
- if ss_cert:
- try:
- ctx = ssl.create_default_context()
- ctx.check_hostname = False
- ctx.verify_mode = ssl.CERT_NONE
- except AttributeError:
- self.error('HTTPS self-signed certificate ignore not enabled')
- else:
- handlers.append(urllib2.HTTPSHandler(context=ctx))
- self.debug("Enabling HTTP self-signed certificate ignore")
-
- # PROXY handler
- if proxy and isinstance(proxy, str) and not ss_cert:
- handlers.append(urllib2.ProxyHandler(dict(http=proxy)))
- self.debug("Enabling HTTP proxy handler (%s)" % proxy)
-
- opener = urllib2.build_opener(*handlers)
- return opener
-
- def _build_opener(self, **kwargs):
+ self.proxy_user = self.configuration.get('proxy_user')
+ self.proxy_password = self.configuration.get('proxy_pass')
+ self.proxy_url = self.configuration.get('proxy_url')
+ self._manager = None
+
+ def __make_headers(self, **header_kw):
+ user = header_kw.get('user') or self.user
+ password = header_kw.get('pass') or self.password
+ proxy_user = header_kw.get('proxy_user') or self.proxy_user
+ proxy_password = header_kw.get('proxy_pass') or self.proxy_password
+ header_params = dict(keep_alive=True)
+ proxy_header_params = dict()
+ if user and password:
+ header_params['basic_auth'] = '{user}:{password}'.format(user=user,
+ password=password)
+ if proxy_user and proxy_password:
+ proxy_header_params['proxy_basic_auth'] = '{user}:{password}'.format(user=proxy_user,
+ password=proxy_password)
try:
- return self.__add_openers(**kwargs)
+ return urllib3.make_headers(**header_params), urllib3.make_headers(**proxy_header_params)
except TypeError as error:
- self.error('build_opener() error:', str(error))
+ self.error('build_header() error: {error}'.format(error=error))
+ return None, None
+
+ def _build_manager(self, **header_kw):
+ header, proxy_header = self.__make_headers(**header_kw)
+ if header is None or proxy_header is None:
+ return None
+ proxy_url = header_kw.get('proxy_url') or self.proxy_url
+ if proxy_url:
+ manager = urllib3.ProxyManager
+ params = dict(proxy_url=proxy_url, headers=header, proxy_headers=proxy_header)
+ else:
+ manager = urllib3.PoolManager
+ params = dict(headers=header)
+ try:
+ return manager(**params)
+ except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error:
+ self.error('build_manager() error:', str(error))
return None
- def _get_raw_data(self, url=None, opener=None):
+ def _get_raw_data(self, url=None, manager=None):
"""
Get raw data from http request
:return: str
"""
- data = None
try:
- opener = opener or self.opener
- data = opener.open(url or self.url, timeout=self.update_every * 2)
- raw_data = data.read().decode('utf-8', 'ignore')
- except urllib2.URLError as error:
- self.error('Url: %s. Error: %s' % (url or self.url, str(error)))
- return None
- except Exception as error:
- self.error(str(error))
+ url = url or self.url
+ manager = manager or self._manager
+ # TODO: timeout, retries and method hardcoded..
+ response = manager.request(method='GET',
+ url=url,
+ timeout=1,
+ retries=1,
+ headers=manager.headers)
+ except (urllib3.exceptions.HTTPError, TypeError, AttributeError) as error:
+ self.error('Url: {url}. Error: {error}'.format(url=url, error=error))
return None
- finally:
- if data is not None:
- data.close()
- return raw_data or None
+ if response.status == 200:
+ return response.data.decode()
+ self.debug('Url: {url}. Http response status code: {code}'.format(url=url, code=response.status))
+ return None
def check(self):
"""
@@ -583,20 +574,21 @@ class UrlService(SimpleService):
self.error('URL is not defined or type is not <str>')
return False
- self.opener = self.__add_openers()
+ self._manager = self._build_manager()
+ if not self._manager:
+ return False
try:
data = self._get_data()
except Exception as error:
- self.error('_get_data() failed. Url: %s. Error: %s' % (self.url, error))
+ self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error))
return False
if isinstance(data, dict) and data:
self._data_from_check = data
return True
- else:
- self.error("_get_data() returned no data or type is not <dict>")
- return False
+ self.error('_get_data() returned no data or type is not <dict>')
+ return False
class SocketService(SimpleService):
@@ -1007,17 +999,20 @@ class MySQLService(SimpleService):
def check(self):
def get_connection_properties(conf, extra_conf):
properties = dict()
- if 'user' in conf and conf['user']:
+ if conf.get('user'):
properties['user'] = conf['user']
- if 'pass' in conf and conf['pass']:
+ if conf.get('pass'):
properties['passwd'] = conf['pass']
- if 'socket' in conf and conf['socket']:
+ if conf.get('socket'):
properties['unix_socket'] = conf['socket']
- elif 'host' in conf and conf['host']:
+ elif conf.get('host'):
properties['host'] = conf['host']
properties['port'] = int(conf.get('port', 3306))
- elif 'my.cnf' in conf and conf['my.cnf']:
- properties['read_default_file'] = conf['my.cnf']
+ elif conf.get('my.cnf'):
+ if MySQLdb.__name__ == 'pymysql':
+ self.error('"my.cnf" parsing is not working for pymysql')
+ else:
+ properties['read_default_file'] = conf['my.cnf']
if isinstance(extra_conf, dict) and extra_conf:
properties.update(extra_conf)
@@ -1046,7 +1041,7 @@ class MySQLService(SimpleService):
log_error('Unsupported "queries" format. Must be not empty <dict>')
return None
- if not PYMYSQL:
+ if not PY_MYSQL:
self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin')
return False