summaryrefslogtreecommitdiffstats
path: root/collectors/python.d.plugin/varnish
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2021-02-07 11:45:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2021-02-07 11:45:55 +0000
commita8220ab2d293bb7f4b014b79d16b2fb05090fa93 (patch)
tree77f0a30f016c0925cf7ee9292e644bba183c2774 /collectors/python.d.plugin/varnish
parentAdding upstream version 1.19.0. (diff)
downloadnetdata-a8220ab2d293bb7f4b014b79d16b2fb05090fa93.tar.xz
netdata-a8220ab2d293bb7f4b014b79d16b2fb05090fa93.zip
Adding upstream version 1.29.0.upstream/1.29.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'collectors/python.d.plugin/varnish')
-rw-r--r--collectors/python.d.plugin/varnish/README.md102
-rw-r--r--collectors/python.d.plugin/varnish/varnish.chart.py145
2 files changed, 150 insertions, 97 deletions
diff --git a/collectors/python.d.plugin/varnish/README.md b/collectors/python.d.plugin/varnish/README.md
index 4de883d31..cb29738f5 100644
--- a/collectors/python.d.plugin/varnish/README.md
+++ b/collectors/python.d.plugin/varnish/README.md
@@ -1,80 +1,56 @@
-# varnish
+<!--
+title: "Varnish Cache monitoring with Netdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/python.d.plugin/varnish/README.md
+sidebar_label: "Varnish Cache"
+-->
-Module uses the `varnishstat` command to provide varnish cache statistics.
+# Varnish Cache monitoring with Netdata
-It produces:
+Provides HTTP accelerator global, Backends (VBE) and Storages (SMF, SMA, MSE) statistics using `varnishstat` tool.
-1. **Connections Statistics** in connections/s
+Note that both, Varnish-Cache (free and open source) and Varnish-Plus (Commercial/Enterprise version), are supported.
- - accepted
- - dropped
+## Requirements
-2. **Client Requests** in requests/s
+- `netdata` user must be a member of the `varnish` group
- - received
+## Charts
-3. **All History Hit Rate Ratio** in percent
+This module produces the following charts:
- - hit
- - miss
- - hitpass
+- Connections Statistics in `connections/s`
+- Client Requests in `requests/s`
+- All History Hit Rate Ratio in `percent`
+- Current Poll Hit Rate Ratio in `percent`
+- Expired Objects in `expired/s`
+- Least Recently Used Nuked Objects in `nuked/s`
+- Number Of Threads In All Pools in `pools`
+- Threads Statistics in `threads/s`
+- Current Queue Length in `requests`
+- Backend Connections Statistics in `connections/s`
+- Requests To The Backend in `requests/s`
+- ESI Statistics in `problems/s`
+- Memory Usage in `MiB`
+- Uptime in `seconds`
-4. **Current Poll Hit Rate Ratio** in percent
+For every backend (VBE):
- - hit
- - miss
- - hitpass
+- Backend Response Statistics in `kilobits/s`
-5. **Expired Objects** in expired/s
+For every storage (SMF, SMA, or MSE):
- - objects
+- Storage Usage in `KiB`
+- Storage Allocated Objects
-6. **Least Recently Used Nuked Objects** in nuked/s
+## Configuration
- - objects
+Edit the `python.d/varnish.conf` configuration file using `edit-config` from the Netdata [config
+directory](/docs/configure/nodes.md), which is typically at `/etc/netdata`.
-7. **Number Of Threads In All Pools** in threads
-
- - threads
-
-8. **Threads Statistics** in threads/s
-
- - created
- - failed
- - limited
-
-9. **Current Queue Length** in requests
-
- - in queue
-
-10. **Backend Connections Statistics** in connections/s
-
- - successful
- - unhealthy
- - reused
- - closed
- - resycled
- - failed
-
-11. **Requests To The Backend** in requests/s
-
- - received
-
-12. **ESI Statistics** in problems/s
-
- - errors
- - warnings
-
-13. **Memory Usage** in MB
-
- - free
- - allocated
-
-14. **Uptime** in seconds
-
- - uptime
-
-## configuration
+```bash
+cd /etc/netdata # Replace this path with your Netdata config directory, if different
+sudo ./edit-config python.d/varnish.conf
+```
Only one parameter is supported:
@@ -82,7 +58,7 @@ Only one parameter is supported:
instance_name: 'name'
```
-The name of the varnishd instance to get logs from. If not specified, the host name is used.
+The name of the `varnishd` instance to get logs from. If not specified, the host name is used.
---
diff --git a/collectors/python.d.plugin/varnish/varnish.chart.py b/collectors/python.d.plugin/varnish/varnish.chart.py
index 58745e24d..534d70926 100644
--- a/collectors/python.d.plugin/varnish/varnish.chart.py
+++ b/collectors/python.d.plugin/varnish/varnish.chart.py
@@ -103,7 +103,7 @@ CHARTS = {
['backend_unhealthy', 'unhealthy', 'incremental'],
['backend_reuse', 'reused', 'incremental'],
['backend_toolate', 'closed', 'incremental'],
- ['backend_recycle', 'resycled', 'incremental'],
+ ['backend_recycle', 'recycled', 'incremental'],
['backend_fail', 'failed', 'incremental']
]
},
@@ -135,9 +135,54 @@ CHARTS = {
}
}
+
+def backend_charts_template(name):
+ order = [
+ '{0}_response_statistics'.format(name),
+ ]
+
+ charts = {
+ order[0]: {
+ 'options': [None, 'Backend "{0}"'.format(name), 'kilobits/s', 'backend response statistics',
+ 'varnish.backend', 'area'],
+ 'lines': [
+ ['{0}_beresp_hdrbytes'.format(name), 'header', 'incremental', 8, 1000],
+ ['{0}_beresp_bodybytes'.format(name), 'body', 'incremental', -8, 1000]
+ ]
+ },
+ }
+
+ return order, charts
+
+
+def storage_charts_template(name):
+ order = [
+ 'storage_{0}_usage'.format(name),
+ 'storage_{0}_alloc_objs'.format(name)
+ ]
+
+ charts = {
+ order[0]: {
+ 'options': [None, 'Storage "{0}" Usage'.format(name), 'KiB', 'storage usage', 'varnish.storage_usage', 'stacked'],
+ 'lines': [
+ ['{0}.g_space'.format(name), 'free', 'absolute', 1, 1 << 10],
+ ['{0}.g_bytes'.format(name), 'allocated', 'absolute', 1, 1 << 10]
+ ]
+ },
+ order[1]: {
+ 'options': [None, 'Storage "{0}" Allocated Objects'.format(name), 'objects', 'storage usage', 'varnish.storage_alloc_objs', 'line'],
+ 'lines': [
+ ['{0}.g_alloc'.format(name), 'allocated', 'absolute']
+ ]
+ }
+ }
+
+ return order, charts
+
+
VARNISHSTAT = 'varnishstat'
-re_version = re.compile(r'varnish-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
+re_version = re.compile(r'varnish-(?:plus-)?(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
class VarnishVersion:
@@ -188,6 +233,8 @@ class Service(ExecutableService):
self.instance_name = configuration.get('instance_name')
self.parser = Parser()
self.command = None
+ self.collected_vbe = set()
+ self.collected_storages = set()
def create_command(self):
varnishstat = find_binary(VARNISHSTAT)
@@ -206,10 +253,7 @@ class Service(ExecutableService):
ver = parse_varnish_version(reply)
if not ver:
self.error("failed to parse reply from '{0}', used regex :'{1}', reply : {2}".format(
- ' '.join(command),
- re_version.pattern,
- reply,
- ))
+ ' '.join(command), re_version.pattern, reply))
return False
if self.instance_name:
@@ -241,9 +285,6 @@ class Service(ExecutableService):
self.error('cant parse the output...')
return False
- if self.parser.re_backend:
- backends = [b[0] for b in self.parser.backend_stats(reply)[::2]]
- self.create_backends_charts(backends)
return True
def get_data(self):
@@ -260,11 +301,11 @@ class Service(ExecutableService):
if not server_stats:
return None
- if self.parser.re_backend:
- backend_stats = self.parser.backend_stats(raw)
- data.update(dict(('_'.join([name, param]), value) for name, param, value in backend_stats))
+ stats = dict((param, value) for _, param, value in server_stats)
+ data.update(stats)
- data.update(dict((param, value) for _, param, value in server_stats))
+ self.get_vbe_backends(data, raw)
+ self.get_storages(server_stats)
# varnish 5 uses default.g_bytes and default.g_space
data['memory_allocated'] = data.get('s0.g_bytes') or data.get('default.g_bytes')
@@ -272,27 +313,63 @@ class Service(ExecutableService):
return data
- def create_backends_charts(self, backends):
- for backend in backends:
- chart_name = ''.join([backend, '_response_statistics'])
- title = 'Backend "{0}"'.format(backend.capitalize())
- hdr_bytes = ''.join([backend, '_beresp_hdrbytes'])
- body_bytes = ''.join([backend, '_beresp_bodybytes'])
-
- chart = {
- chart_name:
- {
- 'options': [None, title, 'kilobits/s', 'backend response statistics',
- 'varnish.backend', 'area'],
- 'lines': [
- [hdr_bytes, 'header', 'incremental', 8, 1000],
- [body_bytes, 'body', 'incremental', -8, 1000]
- ]
- }
- }
-
- self.order.insert(0, chart_name)
- self.definitions.update(chart)
+ def get_vbe_backends(self, data, raw):
+ if not self.parser.re_backend:
+ return
+ stats = self.parser.backend_stats(raw)
+ if not stats:
+ return
+
+ for (name, param, value) in stats:
+ data['_'.join([name, param])] = value
+ if name in self.collected_vbe:
+ continue
+ self.collected_vbe.add(name)
+ self.add_backend_charts(name)
+
+ def get_storages(self, server_stats):
+ # Storage types:
+ # - SMF: File Storage
+ # - SMA: Malloc Storage
+ # - MSE: Massive Storage Engine (Varnish-Plus only)
+ #
+ # Stats example:
+ # [('SMF.', 'ssdStorage.c_req', '47686'),
+ # ('SMF.', 'ssdStorage.c_fail', '0'),
+ # ('SMF.', 'ssdStorage.c_bytes', '668102656'),
+ # ('SMF.', 'ssdStorage.c_freed', '140980224'),
+ # ('SMF.', 'ssdStorage.g_alloc', '39753'),
+ # ('SMF.', 'ssdStorage.g_bytes', '527122432'),
+ # ('SMF.', 'ssdStorage.g_space', '53159968768'),
+ # ('SMF.', 'ssdStorage.g_smf', '40130'),
+ # ('SMF.', 'ssdStorage.g_smf_frag', '311'),
+ # ('SMF.', 'ssdStorage.g_smf_large', '66')]
+ storages = [name for typ, name, _ in server_stats if typ.startswith(('SMF', 'SMA', 'MSE')) and name.endswith('g_space')]
+ if not storages:
+ return
+ for storage in storages:
+ storage = storage.split('.')[0]
+ if storage in self.collected_storages:
+ continue
+ self.collected_storages.add(storage)
+ self.add_storage_charts(storage)
+
+ def add_backend_charts(self, backend_name):
+ self.add_charts(backend_name, backend_charts_template)
+
+ def add_storage_charts(self, storage_name):
+ self.add_charts(storage_name, storage_charts_template)
+
+ def add_charts(self, name, charts_template):
+ order, charts = charts_template(name)
+
+ for chart_name in order:
+ params = [chart_name] + charts[chart_name]['options']
+ dimensions = charts[chart_name]['lines']
+
+ new_chart = self.charts.add_chart(params)
+ for dimension in dimensions:
+ new_chart.add_dimension(dimension)
def parse_varnish_version(lines):