Adding upstream version 1.29.0.upstream/1.29.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 150 insertions, 97 deletions

diff --git a/collectors/python.d.plugin/varnish/README.md b/collectors/python.d.plugin/varnish/README.md
index 4de883d31..cb29738f5 100644
--- a/collectors/python.d.plugin/varnish/README.md
+++ b/collectors/python.d.plugin/varnish/README.md
@@ -1,80 +1,56 @@
-# varnish
+<!--
+title: "Varnish Cache monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/varnish/README.md
+sidebar_label: "Varnish Cache"
+-->
 
-Module uses the `varnishstat` command to provide varnish cache statistics.
+# Varnish Cache monitoring with Netdata
 
-It produces:
+Provides HTTP accelerator global, Backends (VBE) and Storages (SMF, SMA, MSE) statistics using `varnishstat` tool.
 
-1.  **Connections Statistics** in connections/s
+Note that both, Varnish-Cache (free and open source) and Varnish-Plus (Commercial/Enterprise version), are supported.
 
-    -   accepted
-    -   dropped
+## Requirements
 
-2.  **Client Requests** in requests/s
+-   `netdata` user must be a member of the `varnish` group 
 
-    -   received
+## Charts
 
-3.  **All History Hit Rate Ratio** in percent
+This module produces the following charts:
 
-    -   hit
-    -   miss
-    -   hitpass
+-   Connections Statistics in `connections/s`
+-   Client Requests in `requests/s`
+-   All History Hit Rate Ratio in `percent`
+-   Current Poll Hit Rate Ratio in `percent`
+-   Expired Objects in `expired/s`
+-   Least Recently Used Nuked Objects in `nuked/s`
+-   Number Of Threads In All Pools in `pools`
+-   Threads Statistics in `threads/s`
+-   Current Queue Length in `requests`
+-   Backend Connections Statistics in `connections/s`
+-   Requests To The Backend in `requests/s`
+-   ESI Statistics in `problems/s`
+-   Memory Usage in `MiB`
+-   Uptime in `seconds`
 
-4.  **Current Poll Hit Rate Ratio** in percent
+For every backend (VBE):
 
-    -   hit
-    -   miss
-    -   hitpass
+-   Backend Response Statistics in `kilobits/s`
 
-5.  **Expired Objects** in expired/s
+For every storage (SMF, SMA, or MSE):
 
-    -   objects
+-   Storage Usage in `KiB` 
+-   Storage Allocated Objects
 
-6.  **Least Recently Used Nuked Objects** in nuked/s
+## Configuration
 
-    -   objects
+Edit the `python.d/varnish.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
 
-7.  **Number Of Threads In All Pools** in threads
-
-    -   threads
-
-8.  **Threads Statistics** in threads/s
-
-    -   created
-    -   failed
-    -   limited
-
-9.  **Current Queue Length** in requests
-
-    -   in queue
-
-10. **Backend Connections Statistics** in connections/s
-
-    -   successful
-    -   unhealthy
-    -   reused
-    -   closed
-    -   resycled
-    -   failed
-
-11. **Requests To The Backend** in requests/s
-
-    -   received
-
-12. **ESI Statistics** in problems/s
-
-    -   errors
-    -   warnings
-
-13. **Memory Usage** in MB
-
-    -   free
-    -   allocated
-
-14. **Uptime** in seconds
-
-    -   uptime
-
-## configuration
+```bash
+cd /etc/netdata   # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/varnish.conf
+```
 
 Only one parameter is supported:
 
@@ -82,7 +58,7 @@ Only one parameter is supported:
 instance_name: 'name'
 ```
 
-The name of the varnishd instance to get logs from. If not specified, the host name is used.
+The name of the `varnishd` instance to get logs from. If not specified, the host name is used.
 
 ---
 
diff --git a/collectors/python.d.plugin/varnish/varnish.chart.py b/collectors/python.d.plugin/varnish/varnish.chart.py
index 58745e24d..534d70926 100644
--- a/collectors/python.d.plugin/varnish/varnish.chart.py
+++ b/collectors/python.d.plugin/varnish/varnish.chart.py
@@ -103,7 +103,7 @@ CHARTS = {
             ['backend_unhealthy', 'unhealthy', 'incremental'],
             ['backend_reuse', 'reused', 'incremental'],
             ['backend_toolate', 'closed', 'incremental'],
-            ['backend_recycle', 'resycled', 'incremental'],
+            ['backend_recycle', 'recycled', 'incremental'],
             ['backend_fail', 'failed', 'incremental']
         ]
     },
@@ -135,9 +135,54 @@ CHARTS = {
     }
 }
 
+
+def backend_charts_template(name):
+    order = [
+        '{0}_response_statistics'.format(name),
+    ]
+
+    charts = {
+        order[0]: {
+            'options': [None, 'Backend "{0}"'.format(name), 'kilobits/s', 'backend response statistics',
+                        'varnish.backend', 'area'],
+            'lines': [
+                ['{0}_beresp_hdrbytes'.format(name), 'header', 'incremental', 8, 1000],
+                ['{0}_beresp_bodybytes'.format(name), 'body', 'incremental', -8, 1000]
+            ]
+        },
+    }
+
+    return order, charts
+
+
+def storage_charts_template(name):
+    order = [
+        'storage_{0}_usage'.format(name),
+        'storage_{0}_alloc_objs'.format(name)
+    ]
+
+    charts = {
+        order[0]: {
+            'options': [None, 'Storage "{0}" Usage'.format(name), 'KiB', 'storage usage', 'varnish.storage_usage', 'stacked'],
+            'lines': [
+                ['{0}.g_space'.format(name), 'free', 'absolute', 1, 1 << 10],
+                ['{0}.g_bytes'.format(name), 'allocated', 'absolute', 1, 1 << 10]
+            ]
+        },
+        order[1]: {
+            'options': [None, 'Storage "{0}" Allocated Objects'.format(name), 'objects', 'storage usage', 'varnish.storage_alloc_objs', 'line'],
+            'lines': [
+                ['{0}.g_alloc'.format(name), 'allocated', 'absolute']
+            ]
+        }
+    }
+
+    return order, charts
+
+
 VARNISHSTAT = 'varnishstat'
 
-re_version = re.compile(r'varnish-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
+re_version = re.compile(r'varnish-(?:plus-)?(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
 
 
 class VarnishVersion:
@@ -188,6 +233,8 @@ class Service(ExecutableService):
         self.instance_name = configuration.get('instance_name')
         self.parser = Parser()
         self.command = None
+        self.collected_vbe = set()
+        self.collected_storages = set()
 
     def create_command(self):
         varnishstat = find_binary(VARNISHSTAT)
@@ -206,10 +253,7 @@ class Service(ExecutableService):
         ver = parse_varnish_version(reply)
         if not ver:
             self.error("failed to parse reply from '{0}', used regex :'{1}', reply : {2}".format(
-                ' '.join(command),
-                re_version.pattern,
-                reply,
-            ))
+                ' '.join(command), re_version.pattern, reply))
             return False
 
         if self.instance_name:
@@ -241,9 +285,6 @@ class Service(ExecutableService):
             self.error('cant parse the output...')
             return False
 
-        if self.parser.re_backend:
-            backends = [b[0] for b in self.parser.backend_stats(reply)[::2]]
-            self.create_backends_charts(backends)
         return True
 
     def get_data(self):
@@ -260,11 +301,11 @@ class Service(ExecutableService):
         if not server_stats:
             return None
 
-        if self.parser.re_backend:
-            backend_stats = self.parser.backend_stats(raw)
-            data.update(dict(('_'.join([name, param]), value) for name, param, value in backend_stats))
+        stats = dict((param, value) for _, param, value in server_stats)
+        data.update(stats)
 
-        data.update(dict((param, value) for _, param, value in server_stats))
+        self.get_vbe_backends(data, raw)
+        self.get_storages(server_stats)
 
         # varnish 5 uses default.g_bytes and default.g_space
         data['memory_allocated'] = data.get('s0.g_bytes') or data.get('default.g_bytes')
@@ -272,27 +313,63 @@ class Service(ExecutableService):
 
         return data
 
-    def create_backends_charts(self, backends):
-        for backend in backends:
-            chart_name = ''.join([backend, '_response_statistics'])
-            title = 'Backend "{0}"'.format(backend.capitalize())
-            hdr_bytes = ''.join([backend, '_beresp_hdrbytes'])
-            body_bytes = ''.join([backend, '_beresp_bodybytes'])
-
-            chart = {
-                chart_name:
-                {
-                    'options': [None, title, 'kilobits/s', 'backend response statistics',
-                                'varnish.backend', 'area'],
-                    'lines': [
-                        [hdr_bytes, 'header', 'incremental', 8, 1000],
-                        [body_bytes, 'body', 'incremental', -8, 1000]
-                        ]
-                    }
-                }
-
-            self.order.insert(0, chart_name)
-            self.definitions.update(chart)
+    def get_vbe_backends(self, data, raw):
+        if not self.parser.re_backend:
+            return
+        stats = self.parser.backend_stats(raw)
+        if not stats:
+            return
+
+        for (name, param, value) in stats:
+            data['_'.join([name, param])] = value
+            if name in self.collected_vbe:
+                continue
+            self.collected_vbe.add(name)
+            self.add_backend_charts(name)
+
+    def get_storages(self, server_stats):
+        # Storage types:
+        #  - SMF: File Storage
+        #  - SMA: Malloc Storage
+        #  - MSE: Massive Storage Engine (Varnish-Plus only)
+        #
+        # Stats example:
+        #  [('SMF.', 'ssdStorage.c_req', '47686'),
+        #  ('SMF.', 'ssdStorage.c_fail', '0'),
+        #  ('SMF.', 'ssdStorage.c_bytes', '668102656'),
+        #  ('SMF.', 'ssdStorage.c_freed', '140980224'),
+        #  ('SMF.', 'ssdStorage.g_alloc', '39753'),
+        #  ('SMF.', 'ssdStorage.g_bytes', '527122432'),
+        #  ('SMF.', 'ssdStorage.g_space', '53159968768'),
+        #  ('SMF.', 'ssdStorage.g_smf', '40130'),
+        #  ('SMF.', 'ssdStorage.g_smf_frag', '311'),
+        #  ('SMF.', 'ssdStorage.g_smf_large', '66')]
+        storages = [name for typ, name, _ in server_stats if typ.startswith(('SMF', 'SMA', 'MSE')) and name.endswith('g_space')]
+        if not storages:
+            return
+        for storage in storages:
+            storage = storage.split('.')[0]
+            if storage in self.collected_storages:
+                continue
+            self.collected_storages.add(storage)
+            self.add_storage_charts(storage)
+
+    def add_backend_charts(self, backend_name):
+        self.add_charts(backend_name, backend_charts_template)
+
+    def add_storage_charts(self, storage_name):
+        self.add_charts(storage_name, storage_charts_template)
+
+    def add_charts(self, name, charts_template):
+        order, charts = charts_template(name)
+
+        for chart_name in order:
+            params = [chart_name] + charts[chart_name]['options']
+            dimensions = charts[chart_name]['lines']
+
+            new_chart = self.charts.add_chart(params)
+            for dimension in dimensions:
+                new_chart.add_dimension(dimension)
 
 
 def parse_varnish_version(lines):