New upstream version 1.6.0+dfsgupstream/1.6.0+dfsg

2 files changed, 267 insertions, 105 deletions

diff --git a/python.d/python_modules/__init__.py b/python.d/python_modules/__init__.py
index 8d1c8b69c..8d1c8b69c 100755..100644
--- a/python.d/python_modules/__init__.py
+++ b/python.d/python_modules/__init__.py
diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py
index 320c54bae..859300eca 100644
--- a/python.d/python_modules/base.py
+++ b/python.d/python_modules/base.py
@@ -18,19 +18,39 @@
 # using ".encode()" in one thread can block other threads as well (only in python2)
 
 import time
-# import sys
 import os
 import socket
 import select
+import threading
+import msg
+import ssl
+from subprocess import Popen, PIPE
+from sys import exc_info
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
 try:
     import urllib.request as urllib2
 except ImportError:
     import urllib2
 
-from subprocess import Popen, PIPE
+try:
+    import MySQLdb
+    PYMYSQL = True
+except ImportError:
+    try:
+        import pymysql as MySQLdb
+        PYMYSQL = True
+    except ImportError:
+        PYMYSQL = False
 
-import threading
-import msg
+try:
+    PATH = os.getenv('PATH').split(':')
+except AttributeError:
+    PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':')
 
 
 # class BaseService(threading.Thread):
@@ -61,6 +81,7 @@ class SimpleService(threading.Thread):
         self.__first_run = True
         self.order = []
         self.definitions = {}
+        self._data_from_check = dict()
         if configuration is None:
             self.error("BaseService: no configuration parameters supplied. Cannot create Service.")
             raise RuntimeError
@@ -385,7 +406,7 @@ class SimpleService(threading.Thread):
         Create charts
         :return: boolean
         """
-        data = self._get_data()
+        data = self._data_from_check or self._get_data()
         if data is None:
             self.debug("failed to receive data during create().")
             return False
@@ -431,101 +452,93 @@ class SimpleService(threading.Thread):
 
         return updated
 
+    @staticmethod
+    def find_binary(binary):
+        try:
+            if isinstance(binary, str):
+                binary = os.path.basename(binary)
+                return next(('/'.join([p, binary]) for p in PATH
+                            if os.path.isfile('/'.join([p, binary]))
+                            and os.access('/'.join([p, binary]), os.X_OK)))
+            else:
+                return None
+        except StopIteration:
+            return None
+
 
 class UrlService(SimpleService):
-    # TODO add support for https connections
     def __init__(self, configuration=None, name=None):
-        self.url = ""
-        self.user = None
-        self.password = None
-        self.proxies = {}
         SimpleService.__init__(self, configuration=configuration, name=name)
+        self.url = self.configuration.get('url')
+        self.user = self.configuration.get('user')
+        self.password = self.configuration.get('pass')
+        self.ss_cert = self.configuration.get('ss_cert')
 
     def __add_openers(self):
-        # TODO add error handling
-        self.opener = urllib2.build_opener()
-
-        # Proxy handling
-        # TODO currently self.proxies isn't parsed from configuration file
-        # if len(self.proxies) > 0:
-        #     for proxy in self.proxies:
-        #         url = proxy['url']
-        #         # TODO test this:
-        #         if "user" in proxy and "pass" in proxy:
-        #             if url.lower().startswith('https://'):
-        #                 url = 'https://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[8:]
-        #             else:
-        #                 url = 'http://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[7:]
-        #         # FIXME move proxy auth to sth like this:
-        #         #     passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-        #         #     passman.add_password(None, url, proxy['user'], proxy['password'])
-        #         #     opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
-        #
-        #         if url.lower().startswith('https://'):
-        #             opener.add_handler(urllib2.ProxyHandler({'https': url}))
-        #         else:
-        #             opener.add_handler(urllib2.ProxyHandler({'https': url}))
+        def self_signed_cert(ss_cert):
+            if ss_cert:
+                try:
+                    ctx = ssl.create_default_context()
+                    ctx.check_hostname = False
+                    ctx.verify_mode = ssl.CERT_NONE
+                    return urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
+                except AttributeError:
+                    return None
+            else:
+                return None
+
+        self.opener = self_signed_cert(self.ss_cert) or urllib2.build_opener()
 
         # HTTP Basic Auth
-        if self.user is not None and self.password is not None:
+        if self.user and self.password:
+            url_parse = urlparse(self.url)
+            top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
             passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-            passman.add_password(None, self.url, self.user, self.password)
+            passman.add_password(None, top_level_url, self.user, self.password)
             self.opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
             self.debug("Enabling HTTP basic auth")
 
-        #urllib2.install_opener(opener)
-
-    def _get_raw_data(self):
+    def _get_raw_data(self, custom_url=None):
         """
         Get raw data from http request
         :return: str
         """
-        raw = None
+        raw_data = None
+        f = None
         try:
-            f = self.opener.open(self.url, timeout=self.update_every * 2)
-            # f = urllib2.urlopen(self.url, timeout=self.update_every * 2)
-        except Exception as e:
-            self.error(str(e))
+            f = self.opener.open(custom_url or self.url, timeout=self.update_every * 2)
+            raw_data = f.read().decode('utf-8', 'ignore')
+        except Exception as error:
+            self.error('Url: %s. Error: %s' %(custom_url or self.url, str(error)))
             return None
-
-        try:
-            raw = f.read().decode('utf-8', 'ignore')
-        except Exception as e:
-            self.error(str(e))
         finally:
-            f.close()
-        return raw
+            if f is not None: f.close()
+
+        return raw_data or None
 
     def check(self):
         """
         Format configuration data and try to connect to server
         :return: boolean
         """
-        if self.name is None or self.name == str(None):
-            self.name = 'local'
-            self.chart_name += "_" + self.name
-        else:
-            self.name = str(self.name)
-        try:
-            self.url = str(self.configuration['url'])
-        except (KeyError, TypeError):
-            pass
-        try:
-            self.user = str(self.configuration['user'])
-        except (KeyError, TypeError):
-            pass
-        try:
-            self.password = str(self.configuration['pass'])
-        except (KeyError, TypeError):
-            pass
+        if not (self.url and isinstance(self.url, str)):
+            self.error('URL is not defined or type is not <str>')
+            return False
 
         self.__add_openers()
 
-        test = self._get_data()
-        if test is None or len(test) == 0:
+        try:
+            data = self._get_data()
+        except Exception as error:
+            self.error('_get_data() failed. Url: %s. Error: %s' % (self.url, error))
             return False
-        else:
+
+        if isinstance(data, dict) and data:
+            self._data_from_check = data
             return True
+        else:
+            self.error("_get_data() returned no data or type is not <dict>")
+            return False
 
 
 class SocketService(SimpleService):
@@ -829,63 +842,212 @@ class LogService(SimpleService):
 
 
 class ExecutableService(SimpleService):
-    bad_substrings = ('&', '|', ';', '>', '<')
 
     def __init__(self, configuration=None, name=None):
-        self.command = ""
         SimpleService.__init__(self, configuration=configuration, name=name)
+        self.command = None
 
     def _get_raw_data(self):
         """
         Get raw data from executed command
-        :return: str
+        :return: <list>
         """
         try:
             p = Popen(self.command, stdout=PIPE, stderr=PIPE)
-        except Exception as e:
-            self.error("Executing command", self.command, "resulted in error:", str(e))
+        except Exception as error:
+            self.error("Executing command", self.command, "resulted in error:", str(error))
             return None
-        data = []
+        data = list()
         for line in p.stdout.readlines():
-            data.append(str(line.decode()))
+            data.append(line.decode())
 
-        if len(data) == 0:
-            self.error("No data collected.")
-            return None
-
-        return data
+        return data or None
 
     def check(self):
         """
         Parse basic configuration, check if command is whitelisted and is returning values
-        :return: boolean
+        :return: <boolean>
         """
-        if self.name is not None or self.name != str(None):
-            self.name = ""
+        # Preference: 1. "command" from configuration file 2. "command" from plugin (if specified)
+        if 'command' in self.configuration:
+            self.command = self.configuration['command']
+
+        # "command" must be: 1.not None 2. type <str>
+        if not (self.command and isinstance(self.command, str)):
+            self.error('Command is not defined or command type is not <str>')
+            return False
+
+        # Split "command" into: 1. command <str> 2. options <list>
+        command, opts = self.command.split()[0], self.command.split()[1:]
+
+        # Check for "bad" symbols in options. No pipes, redirects etc. TODO: what is missing?
+        bad_opts = set(''.join(opts)) & set(['&', '|', ';', '>', '<'])
+        if bad_opts:
+            self.error("Bad command argument(s): %s" % bad_opts)
+            return False
+
+        # Find absolute path ('echo' => '/bin/echo')
+        if '/' not in command:
+            command = self.find_binary(command)
+            if not command:
+                self.error('Can\'t locate "%s" binary in PATH(%s)' % (self.command, PATH))
+                return False
+        # Check if binary exist and executable
         else:
-            self.name = str(self.name)
+            if not (os.path.isfile(command) and os.access(command, os.X_OK)):
+                self.error('"%s" is not a file or not executable' % command)
+                return False
+
+        self.command = [command] + opts if opts else [command]
+
         try:
-            self.command = str(self.configuration['command'])
-        except (KeyError, TypeError):
-            self.info("No command specified. Using: '" + self.command + "'")
-	# Splitting self.command on every space so subprocess.Popen reads it properly
-        self.command = self.command.split(' ')
+            data = self._get_data()
+        except Exception as error:
+            self.error('_get_data() failed. Command: %s. Error: %s' % (self.command, error))
+            return False
 
-        for arg in self.command[1:]:
-            if any(st in arg for st in self.bad_substrings):
-                self.error("Bad command argument:" + " ".join(self.command[1:]))
-                return False
+        if isinstance(data, dict) and data:
+            # We need this for create() method. No reason to execute get_data() again if result is not empty dict()
+            self._data_from_check = data
+            return True
+        else:
+            self.error("Command", str(self.command), "returned no data")
+            return False
+
+
+class MySQLService(SimpleService):
+
+    def __init__(self, configuration=None, name=None):
+        SimpleService.__init__(self, configuration=configuration, name=name)
+        self.__connection = None
+        self.__conn_properties = dict()
+        self.extra_conn_properties = dict()
+        self.__queries = self.configuration.get('queries', dict())
+        self.queries = dict()
+
+    def __connect(self):
+        try:
+            connection = MySQLdb.connect(connect_timeout=self.update_every, **self.__conn_properties)
+        except (MySQLdb.MySQLError, TypeError, AttributeError) as error:
+            return None, str(error)
+        else:
+            return connection, None
 
-        # test command and search for it in /usr/sbin or /sbin when failed
-        base = self.command[0].split('/')[-1]
-        if self._get_raw_data() is None:
-            for prefix in ['/sbin/', '/usr/sbin/']:
-                self.command[0] = prefix + base
-                if os.path.isfile(self.command[0]):
-                    break
+    def check(self):
+        def get_connection_properties(conf, extra_conf):
+            properties = dict()
+            if 'user' in conf and conf['user']:
+                properties['user'] = conf['user']
+            if 'pass' in conf and conf['pass']:
+                properties['passwd'] = conf['pass']
+            if 'socket' in conf and conf['socket']:
+                properties['unix_socket'] = conf['socket']
+            elif 'host' in conf and conf['host']:
+                properties['host'] = conf['host']
+                properties['port'] = int(conf.get('port', 3306))
+            elif 'my.cnf' in conf and conf['my.cnf']:
+                properties['read_default_file'] = conf['my.cnf']
+            if isinstance(extra_conf, dict) and extra_conf:
+                properties.update(extra_conf)
+
+            return properties or None
+
+        def is_valid_queries_dict(raw_queries, log_error):
+            """
+            :param raw_queries: dict:
+            :param log_error: function:
+            :return: dict or None
+
+            raw_queries is valid when: type <dict> and not empty after is_valid_query(for all queries)
+            """
+            def is_valid_query(query):
+                return all([isinstance(query, str),
+                            query.startswith(('SELECT', 'select', 'SHOW', 'show'))])
+
+            if hasattr(raw_queries, 'keys') and raw_queries:
+                valid_queries = dict([(n, q) for n, q in raw_queries.items() if is_valid_query(q)])
+                bad_queries = set(raw_queries) - set(valid_queries)
+
+                if bad_queries:
+                    log_error('Removed query(s): %s' % bad_queries)
+                return valid_queries
+            else:
+                log_error('Unsupported "queries" format. Must be not empty <dict>')
+                return None
 
-        if self._get_data() is None or len(self._get_data()) == 0:
-            self.error("Command", self.command, "returned no data")
+        if not PYMYSQL:
+            self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin')
             return False
 
-        return True
+        # Preference: 1. "queries" from the configuration file 2. "queries" from the module
+        self.queries = self.__queries or self.queries
+        # Check if "self.queries" exist, not empty and all queries are in valid format
+        self.queries = is_valid_queries_dict(self.queries, self.error)
+        if not self.queries:
+            return None
+
+        # Get connection properties
+        self.__conn_properties = get_connection_properties(self.configuration, self.extra_conn_properties)
+        if not self.__conn_properties:
+            self.error('Connection properties are missing')
+            return False
+
+        # Create connection to the database
+        self.__connection, error = self.__connect()
+        if error:
+            self.error('Can\'t establish connection to MySQL: %s' % error)
+            return False
+
+        try:
+            data = self._get_data() 
+        except Exception as error:
+            self.error('_get_data() failed. Error: %s' % error)
+            return False
+
+        if isinstance(data, dict) and data:
+            # We need this for create() method
+            self._data_from_check = data
+            return True
+        else:
+            self.error("_get_data() returned no data or type is not <dict>")
+            return False
+    
+    def _get_raw_data(self, description=None):
+        """
+        Get raw data from MySQL server
+        :return: dict: fetchall() or (fetchall(), description)
+        """
+
+        if not self.__connection:
+            self.__connection, error = self.__connect()
+            if error:
+                return None
+
+        raw_data = dict()
+        queries = dict(self.queries)
+        try:
+            with self.__connection as cursor:
+                for name, query in queries.items():
+                    try:
+                        cursor.execute(query)
+                    except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as error:
+                        if self.__is_error_critical(err_class=exc_info()[0], err_text=str(error)):
+                            raise RuntimeError
+                        self.error('Removed query: %s[%s]. Error: %s'
+                                   % (name, query, error))
+                        self.queries.pop(name)
+                        continue
+                    else:
+                        raw_data[name] = (cursor.fetchall(), cursor.description) if description else cursor.fetchall()
+            self.__connection.commit()
+        except (MySQLdb.MySQLError, RuntimeError, TypeError, AttributeError):
+            self.__connection.close()
+            self.__connection = None
+            return None
+        else:
+            return raw_data or None
+
+    @staticmethod
+    def __is_error_critical(err_class, err_text):
+        return err_class == MySQLdb.OperationalError and all(['denied' not in err_text,
+                                                              'Unknown column' not in err_text])