diff options
Diffstat (limited to 'python.d/python_modules/base.py')
| -rw-r--r-- | python.d/python_modules/base.py | 372 |
1 files changed, 267 insertions, 105 deletions
diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py index 320c54bae..859300eca 100644 --- a/python.d/python_modules/base.py +++ b/python.d/python_modules/base.py @@ -18,19 +18,39 @@ # using ".encode()" in one thread can block other threads as well (only in python2) import time -# import sys import os import socket import select +import threading +import msg +import ssl +from subprocess import Popen, PIPE +from sys import exc_info + +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + try: import urllib.request as urllib2 except ImportError: import urllib2 -from subprocess import Popen, PIPE +try: + import MySQLdb + PYMYSQL = True +except ImportError: + try: + import pymysql as MySQLdb + PYMYSQL = True + except ImportError: + PYMYSQL = False -import threading -import msg +try: + PATH = os.getenv('PATH').split(':') +except AttributeError: + PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':') # class BaseService(threading.Thread): @@ -61,6 +81,7 @@ class SimpleService(threading.Thread): self.__first_run = True self.order = [] self.definitions = {} + self._data_from_check = dict() if configuration is None: self.error("BaseService: no configuration parameters supplied. Cannot create Service.") raise RuntimeError @@ -385,7 +406,7 @@ class SimpleService(threading.Thread): Create charts :return: boolean """ - data = self._get_data() + data = self._data_from_check or self._get_data() if data is None: self.debug("failed to receive data during create().") return False @@ -431,101 +452,93 @@ class SimpleService(threading.Thread): return updated + @staticmethod + def find_binary(binary): + try: + if isinstance(binary, str): + binary = os.path.basename(binary) + return next(('/'.join([p, binary]) for p in PATH + if os.path.isfile('/'.join([p, binary])) + and os.access('/'.join([p, binary]), os.X_OK))) + else: + return None + except StopIteration: + return None + class UrlService(SimpleService): - # TODO add support for https connections def __init__(self, configuration=None, name=None): - self.url = "" - self.user = None - self.password = None - self.proxies = {} SimpleService.__init__(self, configuration=configuration, name=name) + self.url = self.configuration.get('url') + self.user = self.configuration.get('user') + self.password = self.configuration.get('pass') + self.ss_cert = self.configuration.get('ss_cert') def __add_openers(self): - # TODO add error handling - self.opener = urllib2.build_opener() - - # Proxy handling - # TODO currently self.proxies isn't parsed from configuration file - # if len(self.proxies) > 0: - # for proxy in self.proxies: - # url = proxy['url'] - # # TODO test this: - # if "user" in proxy and "pass" in proxy: - # if url.lower().startswith('https://'): - # url = 'https://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[8:] - # else: - # url = 'http://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[7:] - # # FIXME move proxy auth to sth like this: - # # passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - # # passman.add_password(None, url, proxy['user'], proxy['password']) - # # opener.add_handler(urllib2.HTTPBasicAuthHandler(passman)) - # - # if url.lower().startswith('https://'): - # opener.add_handler(urllib2.ProxyHandler({'https': url})) - # else: - # opener.add_handler(urllib2.ProxyHandler({'https': url})) + def self_signed_cert(ss_cert): + if ss_cert: + try: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return urllib2.build_opener(urllib2.HTTPSHandler(context=ctx)) + except AttributeError: + return None + else: + return None + + self.opener = self_signed_cert(self.ss_cert) or urllib2.build_opener() # HTTP Basic Auth - if self.user is not None and self.password is not None: + if self.user and self.password: + url_parse = urlparse(self.url) + top_level_url = '://'.join([url_parse.scheme, url_parse.netloc]) passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, self.url, self.user, self.password) + passman.add_password(None, top_level_url, self.user, self.password) self.opener.add_handler(urllib2.HTTPBasicAuthHandler(passman)) self.debug("Enabling HTTP basic auth") - #urllib2.install_opener(opener) - - def _get_raw_data(self): + def _get_raw_data(self, custom_url=None): """ Get raw data from http request :return: str """ - raw = None + raw_data = None + f = None try: - f = self.opener.open(self.url, timeout=self.update_every * 2) - # f = urllib2.urlopen(self.url, timeout=self.update_every * 2) - except Exception as e: - self.error(str(e)) + f = self.opener.open(custom_url or self.url, timeout=self.update_every * 2) + raw_data = f.read().decode('utf-8', 'ignore') + except Exception as error: + self.error('Url: %s. Error: %s' %(custom_url or self.url, str(error))) return None - - try: - raw = f.read().decode('utf-8', 'ignore') - except Exception as e: - self.error(str(e)) finally: - f.close() - return raw + if f is not None: f.close() + + return raw_data or None def check(self): """ Format configuration data and try to connect to server :return: boolean """ - if self.name is None or self.name == str(None): - self.name = 'local' - self.chart_name += "_" + self.name - else: - self.name = str(self.name) - try: - self.url = str(self.configuration['url']) - except (KeyError, TypeError): - pass - try: - self.user = str(self.configuration['user']) - except (KeyError, TypeError): - pass - try: - self.password = str(self.configuration['pass']) - except (KeyError, TypeError): - pass + if not (self.url and isinstance(self.url, str)): + self.error('URL is not defined or type is not <str>') + return False self.__add_openers() - test = self._get_data() - if test is None or len(test) == 0: + try: + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Url: %s. Error: %s' % (self.url, error)) return False - else: + + if isinstance(data, dict) and data: + self._data_from_check = data return True + else: + self.error("_get_data() returned no data or type is not <dict>") + return False class SocketService(SimpleService): @@ -829,63 +842,212 @@ class LogService(SimpleService): class ExecutableService(SimpleService): - bad_substrings = ('&', '|', ';', '>', '<') def __init__(self, configuration=None, name=None): - self.command = "" SimpleService.__init__(self, configuration=configuration, name=name) + self.command = None def _get_raw_data(self): """ Get raw data from executed command - :return: str + :return: <list> """ try: p = Popen(self.command, stdout=PIPE, stderr=PIPE) - except Exception as e: - self.error("Executing command", self.command, "resulted in error:", str(e)) + except Exception as error: + self.error("Executing command", self.command, "resulted in error:", str(error)) return None - data = [] + data = list() for line in p.stdout.readlines(): - data.append(str(line.decode())) + data.append(line.decode()) - if len(data) == 0: - self.error("No data collected.") - return None - - return data + return data or None def check(self): """ Parse basic configuration, check if command is whitelisted and is returning values - :return: boolean + :return: <boolean> """ - if self.name is not None or self.name != str(None): - self.name = "" + # Preference: 1. "command" from configuration file 2. "command" from plugin (if specified) + if 'command' in self.configuration: + self.command = self.configuration['command'] + + # "command" must be: 1.not None 2. type <str> + if not (self.command and isinstance(self.command, str)): + self.error('Command is not defined or command type is not <str>') + return False + + # Split "command" into: 1. command <str> 2. options <list> + command, opts = self.command.split()[0], self.command.split()[1:] + + # Check for "bad" symbols in options. No pipes, redirects etc. TODO: what is missing? + bad_opts = set(''.join(opts)) & set(['&', '|', ';', '>', '<']) + if bad_opts: + self.error("Bad command argument(s): %s" % bad_opts) + return False + + # Find absolute path ('echo' => '/bin/echo') + if '/' not in command: + command = self.find_binary(command) + if not command: + self.error('Can\'t locate "%s" binary in PATH(%s)' % (self.command, PATH)) + return False + # Check if binary exist and executable else: - self.name = str(self.name) + if not (os.path.isfile(command) and os.access(command, os.X_OK)): + self.error('"%s" is not a file or not executable' % command) + return False + + self.command = [command] + opts if opts else [command] + try: - self.command = str(self.configuration['command']) - except (KeyError, TypeError): - self.info("No command specified. Using: '" + self.command + "'") - # Splitting self.command on every space so subprocess.Popen reads it properly - self.command = self.command.split(' ') + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Command: %s. Error: %s' % (self.command, error)) + return False - for arg in self.command[1:]: - if any(st in arg for st in self.bad_substrings): - self.error("Bad command argument:" + " ".join(self.command[1:])) - return False + if isinstance(data, dict) and data: + # We need this for create() method. No reason to execute get_data() again if result is not empty dict() + self._data_from_check = data + return True + else: + self.error("Command", str(self.command), "returned no data") + return False + + +class MySQLService(SimpleService): + + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.__connection = None + self.__conn_properties = dict() + self.extra_conn_properties = dict() + self.__queries = self.configuration.get('queries', dict()) + self.queries = dict() + + def __connect(self): + try: + connection = MySQLdb.connect(connect_timeout=self.update_every, **self.__conn_properties) + except (MySQLdb.MySQLError, TypeError, AttributeError) as error: + return None, str(error) + else: + return connection, None - # test command and search for it in /usr/sbin or /sbin when failed - base = self.command[0].split('/')[-1] - if self._get_raw_data() is None: - for prefix in ['/sbin/', '/usr/sbin/']: - self.command[0] = prefix + base - if os.path.isfile(self.command[0]): - break + def check(self): + def get_connection_properties(conf, extra_conf): + properties = dict() + if 'user' in conf and conf['user']: + properties['user'] = conf['user'] + if 'pass' in conf and conf['pass']: + properties['passwd'] = conf['pass'] + if 'socket' in conf and conf['socket']: + properties['unix_socket'] = conf['socket'] + elif 'host' in conf and conf['host']: + properties['host'] = conf['host'] + properties['port'] = int(conf.get('port', 3306)) + elif 'my.cnf' in conf and conf['my.cnf']: + properties['read_default_file'] = conf['my.cnf'] + if isinstance(extra_conf, dict) and extra_conf: + properties.update(extra_conf) + + return properties or None + + def is_valid_queries_dict(raw_queries, log_error): + """ + :param raw_queries: dict: + :param log_error: function: + :return: dict or None + + raw_queries is valid when: type <dict> and not empty after is_valid_query(for all queries) + """ + def is_valid_query(query): + return all([isinstance(query, str), + query.startswith(('SELECT', 'select', 'SHOW', 'show'))]) + + if hasattr(raw_queries, 'keys') and raw_queries: + valid_queries = dict([(n, q) for n, q in raw_queries.items() if is_valid_query(q)]) + bad_queries = set(raw_queries) - set(valid_queries) + + if bad_queries: + log_error('Removed query(s): %s' % bad_queries) + return valid_queries + else: + log_error('Unsupported "queries" format. Must be not empty <dict>') + return None - if self._get_data() is None or len(self._get_data()) == 0: - self.error("Command", self.command, "returned no data") + if not PYMYSQL: + self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin') return False - return True + # Preference: 1. "queries" from the configuration file 2. "queries" from the module + self.queries = self.__queries or self.queries + # Check if "self.queries" exist, not empty and all queries are in valid format + self.queries = is_valid_queries_dict(self.queries, self.error) + if not self.queries: + return None + + # Get connection properties + self.__conn_properties = get_connection_properties(self.configuration, self.extra_conn_properties) + if not self.__conn_properties: + self.error('Connection properties are missing') + return False + + # Create connection to the database + self.__connection, error = self.__connect() + if error: + self.error('Can\'t establish connection to MySQL: %s' % error) + return False + + try: + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Error: %s' % error) + return False + + if isinstance(data, dict) and data: + # We need this for create() method + self._data_from_check = data + return True + else: + self.error("_get_data() returned no data or type is not <dict>") + return False + + def _get_raw_data(self, description=None): + """ + Get raw data from MySQL server + :return: dict: fetchall() or (fetchall(), description) + """ + + if not self.__connection: + self.__connection, error = self.__connect() + if error: + return None + + raw_data = dict() + queries = dict(self.queries) + try: + with self.__connection as cursor: + for name, query in queries.items(): + try: + cursor.execute(query) + except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as error: + if self.__is_error_critical(err_class=exc_info()[0], err_text=str(error)): + raise RuntimeError + self.error('Removed query: %s[%s]. Error: %s' + % (name, query, error)) + self.queries.pop(name) + continue + else: + raw_data[name] = (cursor.fetchall(), cursor.description) if description else cursor.fetchall() + self.__connection.commit() + except (MySQLdb.MySQLError, RuntimeError, TypeError, AttributeError): + self.__connection.close() + self.__connection = None + return None + else: + return raw_data or None + + @staticmethod + def __is_error_critical(err_class, err_text): + return err_class == MySQLdb.OperationalError and all(['denied' not in err_text, + 'Unknown column' not in err_text]) |